From 5e0f238406d5d8dfc8b69ca79ee9e72be7d91a06 Mon Sep 17 00:00:00 2001
From: Ben Singleton <benfsingleton@gmail.com>
Date: Sat, 4 Nov 2023 08:08:55 -0400
Subject: [PATCH 1/6] misc updates

---
 docs/integrations/ib.md                       | 234 +++++++++++++++---
 docs/tutorials/backtest_high_level.md         |   4 +-
 .../notebooks/external_data_backtest.ipynb    |   4 +-
 .../adapters/interactive_brokers/data.py      |   3 +-
 .../adapters/interactive_brokers/execution.py |   3 +-
 .../adapters/interactive_brokers/gateway.py   |   4 +-
 .../parsing/instruments.py                    |   7 +
 nautilus_trader/persistence/loaders.py        |  78 ++----
 nautilus_trader/persistence/wranglers_v2.py   |  57 +++--
 nautilus_trader/test_kit/providers.py         |  14 +-
 10 files changed, 278 insertions(+), 130 deletions(-)

diff --git a/docs/integrations/ib.md b/docs/integrations/ib.md
index 4cc5bd9a2e98..f654efabf57e 100644
--- a/docs/integrations/ib.md
+++ b/docs/integrations/ib.md
@@ -1,21 +1,208 @@
 # Interactive Brokers
 
-NautilusTrader offers an adapter for integrating with the Interactive Brokers Gateway via 
-[ibapi](https://github.com/nautechsystems/ibapi).
+Interactive Brokers (IB) is a trading platform where you can trade stocks, options, futures, currencies, bonds, funds, and crypto. NautilusTrader provides an adapter to integrate with IB using their [Trader Workstation (TWS) API](https://interactivebrokers.github.io/tws-api/index.html) via their Python library, [ibapi](https://github.com/nautechsystems/ibapi).
 
-**Note**: If you are planning on using the built-in docker TWS Gateway when using the Interactive Brokers adapter,
-you must ensure the `docker` package is installed. Run `poetry install --extras "ib docker"` 
-or `poetry install --all-extras` inside your environment to ensure the necessary packages are installed.
+The TWS API is an interface to IB's standalone trading applications, TWS and IB Gateway, which can be downloaded on IB's website. If you have not already installed TWS or IB Gateway, follow Interactive Brokers' [Initial Setup](https://interactivebrokers.github.io/tws-api/initial_setup.html) guide. You will define a connection to either of these two applications in NautilusTrader's `InteractiveBrokersClient`. 
+
+Another (and perhaps easier way) to get started is to use a [dockerized version](https://github.com/unusualalpha/ib-gateway-docker/pkgs/container/ib-gateway) of IB Gateway, which is also what you would use when deploying your trading strategies on a hosted cloud platform. You will need [Docker](https://www.docker.com/) installed on your machine and the [docker](https://pypi.org/project/docker/) Python package, which is conveniently bundled with NautilusTrader as an extra package.
+
+**Note**: The standalone TWS and IB Gateway applications require human intervention to specify a username, password and trading mode (live or paper trading) at startup. The dockerized IB Gateway is able to do so programmatically.
+
+## Installation
+
+To install the latest nautilus-trader package with the `ibapi` and optional `docker` extra dependencies using pip, run:
+
+```
+pip install -U "nautilus_trader[ib,docker]"
+```
+
+To install using poetry, run:
+
+```
+poetry add "nautilus_trader[ib,docker]"
+```
+
+**Note**: IB does not provide wheels for `ibapi`, so Nautilus [repackages]( https://pypi.org/project/nautilus-ibapi/) and releases it to PyPI.
+
+## Getting Started
+
+Before writing strategies, TWS / IB Gateway needs to be running. Launch either of the two standalone applications and enter your credentials, or start the dockerized IB Gateway using `InteractiveBrokersGateway` (make sure Docker is running in the background already!):
+
+```
+from nautilus_trader.adapters.interactive_brokers.gateway import InteractiveBrokersGateway
+
+# This may take a short while to start up, especially the first time
+gateway = InteractiveBrokersGateway(username="test", password="test", start=True)
+
+# Confirm you are logged in
+print(gateway.is_logged_in(gateway.container))
+
+# Inspect the logs
+print(gateway.container.logs())
+```
+
+**Note**: There are two options for supplying your credentials to the Interactive Brokers Gateway, Exec and Data clients.
+Either pass the corresponding `username` and `password` values to the config dictionaries, or
+set the following environment variables: 
+- `TWS_USERNAME`
+- `TWS_PASSWORD`
 
 ## Overview
 
-The following integration classes are available:
-- `InteractiveBrokersInstrumentProvider` which allows querying Interactive Brokers for instruments.
-- `InteractiveBrokersDataClient` which connects to the `Gateway` and streams market data.
-- `InteractiveBrokersExecutionClient` which allows the retrieval of account information and execution of orders.
+The adapter is comprised of the following major components:
+- `InteractiveBrokersClient` which uses `ibapi` to execute TWS API requests, supporting all other integration classes.
+- `HistoricInteractiveBrokersClient` which provides a straightforward way to retrieve instruments and historical bar and tick data to load into the catalog (generally for backtesting).
+- `InteractiveBrokersInstrumentProvider` which retrieves or queries instruments for trading.
+- `InteractiveBrokersDataClient` which streams market data for trading.
+- `InteractiveBrokersExecutionClient` which retrieves account information and executes orders for trading.
+
+## Instruments & Contracts
+
+In Interactive Brokers, the concept of a NautilusTrader `Instrument` is called a [Contract](https://interactivebrokers.github.io/tws-api/contracts.html). A Contract can be represented in two ways: a [basic contract](https://interactivebrokers.github.io/tws-api/classIBApi_1_1Contract.html) or a [detailed contract](https://interactivebrokers.github.io/tws-api/classIBApi_1_1ContractDetails.html), which are defined in the adapter using the classes `IBContract` and `IBContractDetails`, respectively. Contract details include important information like supported order types and trading hours, which aren't found in the basic contract. As a result, `IBContractDetails` can be converted to an `Instrument` but `IBContract` cannot.
+
+To find basic contract information, use the [IB Contract Information Center](https://pennies.interactivebrokers.com/cstools/contract_info/). 
+
+Examples of `IBContracts`:
+```
+from nautilus_trader.adapters.interactive_brokers.common import IBContract
+ 
+# Stock
+IBContract(secType='STK', exchange='SMART', primaryExchange='ARCA', symbol='SPY')
+
+# Bond
+IBContract(secType='BOND', secIdType='ISIN', secId='US03076KAA60')
+
+# Option
+IBContract(secType='STK', exchange='SMART', primaryExchange='ARCA', symbol='SPY', lastTradeDateOrContractMonth='20251219', build_options_chain=True)
+
+# CFD
+IBContract(secType='CFD', symbol='IBUS30')
+
+# Future
+IBContract(secType='CONTFUT', exchange='CME', symbol='ES', build_futures_chain=True)
+
+# Forex
+IBContract(secType='CASH', exchange='IDEALPRO', symbol='EUR', currency='GBP')
+
+# Crypto
+IBContract(secType='CRYPTO', symbol='ETH', exchange='PAXOS', currency='USD')
+```
+
+## Historical Data & Backtesting
 
-## Instruments
-Interactive Brokers allows searching for instruments via the `qualifyContracts` API, which, if given enough information
+The first step in developing strategies using the IB adapter typically involves fetching historical data that can be used for backtesting. The `HistoricInteractiveBrokersClient` provides a number of convenience methods to request data and save it to the catalog.
+
+The following example illustrates retrieving and saving instrument, bar, quote tick, and trade tick data. Read more about using the data for backtesting [here].
+
+```
+# Define path to existing catalog or desired new catalog path
+CATALOG_PATH = "./catalog"
+
+async def main():
+    contract = IBContract(
+        secType="STK",
+        symbol="AAPL",
+        exchange="SMART",
+        primaryExchange="NASDAQ",
+    )
+    client = HistoricInteractiveBrokersClient(
+        catalog=ParquetDataCatalog(CATALOG_PATH)
+    )
+    # By default, data is written to the catalog
+    await client.get_instruments(contract=contract)
+
+    # All the methods also return the retrieved objects
+    # so you can introspect them or explicitly write them
+    # to the catalog
+    bars = await client.get_historical_bars(
+        contract=contract,
+        bar_type=...
+        write_to_catalog=False
+    )
+    print(bars)
+    client.catalog.write_data(bars)
+    
+    await client.get_historical_ticks(
+        contract=contract,
+        bar_type..
+    )
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## Live Trading
+
+To live trade or paper trade, a `TradingNode` needs to be built and run with a `InteractiveBrokersDataClient` and a `InteractiveBrokersExecutionClient`, which both rely on a `InteractiveBrokersInstrumentProvider`.
+
+### InstrumentProvider
+
+To retrieve instruments, you will need to use the `InteractiveBrokersInstrumentProvider`. This provider is also used under the hood in the `HistoricInteractiveBrokersClient` to retrieve instruments for data collection.
+
+```
+from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersDataClientConfig
+
+gateway_config = InteractiveBrokersGatewayConfig(username="test", password="test")
+
+# Specify instruments to retrieve in load_ids and / or load_contracts
+# The following parameters should only be specified if retrieving futures or options: build_futures_chain, build_options_chain, min_expiry_days, max_expiry_days
+instrument_provider = InteractiveBrokersInstrumentProviderConfig(
+    build_futures_chain=False,  # optional, only if fetching futures
+    build_options_chain=False,  # optional, only if fetching futures
+    min_expiry_days=10,  # optional, only if fetching futures / options
+    max_expiry_days=60,  # optional, only if fetching futures / options
+    load_ids=frozenset(
+        [
+            "EUR/USD.IDEALPRO",
+            "BTC/USD.PAXOS",
+            "SPY.ARCA",
+            "V.NYSE",
+            "YMH24.CBOT",
+            "CLZ27.NYMEX",
+            "ESZ27.CME",
+        ],
+    ),
+    load_contracts=frozenset(ib_contracts),
+)
+
+# Configure the trading node
+config_node = TradingNodeConfig(
+    trader_id="IB-001",
+    logging=LoggingConfig(log_level="INFO"),
+    data_clients={
+        "IB": InteractiveBrokersDataClientConfig(
+            ibg_host="127.0.0.1",
+            ibg_port=4002,
+            ibg_client_id=1,
+            handle_revised_bars=False,
+            use_regular_trading_hours=True,
+            market_data_type=IBMarketDataTypeEnum.DELAYED_FROZEN,  # https://interactivebrokers.github.io/tws-api/market_data_type.html
+            instrument_provider=instrument_provider,
+            gateway=gateway,
+        ),
+    },
+    timeout_connection=90.0,
+)
+
+node = TradingNode(config=config_node)
+
+node.trader.add_actor(downloader)
+
+# Register your client factories with the node (can take user defined factories)
+node.add_data_client_factory("InteractiveBrokers", InteractiveBrokersLiveDataClientFactory)
+node.add_exec_client_factory("InteractiveBrokers", InteractiveBrokersLiveExecClientFactory)
+node.build()
+
+# Stop and dispose of the node with SIGINT/CTRL+C
+if __name__ == "__main__":
+    try:
+        node.run()
+    finally:
+        node.dispose()
+
+```
+
+Interactive Brokers allows searching for instruments via the `reqMatchingSymbols` API ([docs](https://interactivebrokers.github.io/tws-api/matching_symbols.html)), which, if given enough information
 can usually resolve a filter into an actual contract(s). A node can request instruments to be loaded by passing 
 configuration to the `InstrumentProviderConfig` when initialising a `TradingNodeConfig` (note that while `filters`
 is a dict, it must be converted to a tuple when passed to `InstrumentProviderConfig`).
@@ -43,18 +230,13 @@ config_node = TradingNodeConfig(
 )
 ```
 
-### Examples queries
-- Stock: `IBContract(secType='STK', exchange='SMART', symbol='AMD', currency='USD')`
-- Stock: `IBContract(secType='STK', exchange='SMART', primaryExchange='NASDAQ', symbol='INTC')`
-- Forex: `InstrumentId('EUR/USD.IDEALPRO')`, `InstrumentId('USD/JPY.IDEALPRO')`
-- CFD: `IBContract(secType='CFD', symbol='IBUS30')`
-- Future: `InstrumentId("YMH24.CBOT")`, `InstrumentId("CLZ27.NYMEX")`, `InstrumentId("ESZ27.CME")`, `InstrumentId('ES.CME')`, `IBContract(secType='CONTFUT', exchange='CME', symbol='ES', build_futures_chain=True)`
-- Option: `InstrumentId('SPY251219C00395000.SMART')`, `IBContract(secType='STK', exchange='SMART', primaryExchange='ARCA', symbol='SPY', lastTradeDateOrContractMonth='20251219', build_options_chain=True)`
-- Bond: `IBContract(secType='BOND', secIdType='ISIN', secId='US03076KAA60')`
-- Crypto: `InstrumentId('BTC/USD.PAXOS')`
+### Data Client
+- `InteractiveBrokersDataClient` which streams market data.
 
+### Execution Client
+- `InteractiveBrokersExecutionClient` which retrieves account information and executes orders.
 
-## Configuration
+### Full Configuration
 The most common use case is to configure a live `TradingNode` to include Interactive Brokers
 data and execution clients. To achieve this, add an `IB` section to your client
 configuration(s) and set the environment variables to your TWS (Traders Workstation) credentials:
@@ -98,13 +280,3 @@ node.add_exec_client_factory("IB", InteractiveBrokersLiveExecClientFactory)
 # Finally build the node
 node.build()
 ```
-
-### API credentials
-There are two options for supplying your credentials to the Interactive Brokers clients.
-Either pass the corresponding `username` and `password` values to the config dictionaries, or
-set the following environment variables: 
-- `TWS_USERNAME`
-- `TWS_PASSWORD`
-
-When starting the trading node, you'll receive immediate confirmation of whether your
-credentials are valid and have trading permissions.
diff --git a/docs/tutorials/backtest_high_level.md b/docs/tutorials/backtest_high_level.md
index 6162fb256125..29e6baa5b20e 100644
--- a/docs/tutorials/backtest_high_level.md
+++ b/docs/tutorials/backtest_high_level.md
@@ -29,7 +29,7 @@ from nautilus_trader.backtest.node import BacktestNode, BacktestVenueConfig, Bac
 from nautilus_trader.config.common import ImportableStrategyConfig
 from nautilus_trader.persistence.catalog import ParquetDataCatalog
 from nautilus_trader.persistence.wranglers import QuoteTickDataWrangler
-from nautilus_trader.test_kit.providers import CSVTickDataLoader
+from nautilus_trader.persistence.loaders import CSVDataLoader
 from nautilus_trader.test_kit.providers import TestInstrumentProvider
 ```
 
@@ -64,7 +64,7 @@ Then we can create Nautilus `QuoteTick` objects by processing the DataFrame with
 
 ```python
 # Here we just take the first data file found and load into a pandas DataFrame
-df = CSVTickDataLoader.load(raw_files[0], index_col=0, format="%Y%m%d %H%M%S%f")
+df = CSVDataLoader.load(raw_files[0], timestamp_column=0, format="%Y%m%d %H%M%S%f")
 df.columns = ["bid_price", "ask_price"]
 
 # Process quote ticks using a wrangler
diff --git a/examples/notebooks/external_data_backtest.ipynb b/examples/notebooks/external_data_backtest.ipynb
index c3a26dffa2ff..e3b1f9d73161 100644
--- a/examples/notebooks/external_data_backtest.ipynb
+++ b/examples/notebooks/external_data_backtest.ipynb
@@ -38,7 +38,7 @@
     "from nautilus_trader.config.common import ImportableStrategyConfig\n",
     "from nautilus_trader.persistence.catalog import ParquetDataCatalog\n",
     "from nautilus_trader.persistence.wranglers import QuoteTickDataWrangler\n",
-    "from nautilus_trader.test_kit.providers import CSVTickDataLoader\n",
+    "from nautilus_trader.test_kit.providers import CSVDataLoader\n",
     "from nautilus_trader.test_kit.providers import TestInstrumentProvider"
    ]
   },
@@ -73,7 +73,7 @@
    "outputs": [],
    "source": [
     "# Here we just take the first data file found and load into a pandas DataFrame\n",
-    "df = CSVTickDataLoader.load(raw_files[0], index_col=0, format=\"%Y%m%d %H%M%S%f\")\n",
+    "df = CSVDataLoader.load(raw_files[0], index_col=0, format=\"%Y%m%d %H%M%S%f\")\n",
     "df.columns = [\"bid_price\", \"ask_price\"]\n",
     "\n",
     "# Process quote ticks using a wrangler\n",
diff --git a/nautilus_trader/adapters/interactive_brokers/data.py b/nautilus_trader/adapters/interactive_brokers/data.py
index afe66ae0c816..01af35c7ce2c 100644
--- a/nautilus_trader/adapters/interactive_brokers/data.py
+++ b/nautilus_trader/adapters/interactive_brokers/data.py
@@ -49,7 +49,8 @@
 
 class InteractiveBrokersDataClient(LiveMarketDataClient):
     """
-    Provides a data client for the InteractiveBrokers exchange.
+    Provides a data client for the InteractiveBrokers exchange by using the `Gateway` to
+    stream market data.
     """
 
     def __init__(
diff --git a/nautilus_trader/adapters/interactive_brokers/execution.py b/nautilus_trader/adapters/interactive_brokers/execution.py
index bfa5ed5ba905..dff5ec7b27e5 100644
--- a/nautilus_trader/adapters/interactive_brokers/execution.py
+++ b/nautilus_trader/adapters/interactive_brokers/execution.py
@@ -96,7 +96,8 @@
 
 class InteractiveBrokersExecutionClient(LiveExecutionClient):
     """
-    Provides an execution client for Interactive Brokers TWS API.
+    Provides an execution client for Interactive Brokers TWS API, allowing for the
+    retrieval of account information and execution of orders.
 
     Parameters
     ----------
diff --git a/nautilus_trader/adapters/interactive_brokers/gateway.py b/nautilus_trader/adapters/interactive_brokers/gateway.py
index 375b609afd93..3cd2a621d546 100644
--- a/nautilus_trader/adapters/interactive_brokers/gateway.py
+++ b/nautilus_trader/adapters/interactive_brokers/gateway.py
@@ -51,8 +51,8 @@ class InteractiveBrokersGateway:
 
     def __init__(
         self,
-        username: str,
-        password: str,
+        username: str | None = None,
+        password: str | None = None,
         host: str | None = "localhost",
         port: int | None = None,
         trading_mode: str | None = "paper",
diff --git a/nautilus_trader/adapters/interactive_brokers/parsing/instruments.py b/nautilus_trader/adapters/interactive_brokers/parsing/instruments.py
index caeb4091787d..d1e8092d5935 100644
--- a/nautilus_trader/adapters/interactive_brokers/parsing/instruments.py
+++ b/nautilus_trader/adapters/interactive_brokers/parsing/instruments.py
@@ -19,6 +19,7 @@
 from decimal import Decimal
 
 import msgspec
+from ibapi.contract import ContractDetails
 
 # fmt: off
 from nautilus_trader.adapters.interactive_brokers.common import IBContract
@@ -116,6 +117,12 @@ def sec_type_to_asset_class(sec_type: str):
     return asset_class_from_str(mapping.get(sec_type, sec_type))
 
 
+def contract_details_to_ib_contract_details(details: ContractDetails) -> IBContractDetails:
+    details.contract = IBContract(**details.contract.__dict__)
+    details = IBContractDetails(**details.__dict__)
+    return details
+
+
 def parse_instrument(
     contract_details: IBContractDetails,
 ) -> Instrument:
diff --git a/nautilus_trader/persistence/loaders.py b/nautilus_trader/persistence/loaders.py
index 06dfd4f05015..f8ee0451cbce 100644
--- a/nautilus_trader/persistence/loaders.py
+++ b/nautilus_trader/persistence/loaders.py
@@ -18,26 +18,26 @@
 import pandas as pd
 
 
-class CSVTickDataLoader:
+class CSVDataLoader:
     """
-    Provides a generic tick data CSV file loader.
+    Loads a CSV file to a `pandas.DataFrame`.
     """
 
     @staticmethod
     def load(
         file_path: PathLike[str] | str,
-        index_col: str | int = "timestamp",
+        timestamp_column: str | int = "timestamp",
         format: str = "mixed",
     ) -> pd.DataFrame:
         """
-        Return a tick `pandas.DataFrame` loaded from the given CSV `file_path`.
+        Return a `pandas.DataFrame` loaded from the given CSV `file_path`.
 
         Parameters
         ----------
         file_path : str, path object or file-like object
             The path to the CSV file.
-        index_col : str | int, default 'timestamp'
-            The index column.
+        timestamp_column : str | int, default 'timestamp'
+            Name of the timestamp column in the CSV file
         format : str, default 'mixed'
             The timestamp column format.
 
@@ -48,54 +48,26 @@ def load(
         """
         df = pd.read_csv(
             file_path,
-            index_col=index_col,
+            index_col=timestamp_column,
             parse_dates=True,
         )
         df.index = pd.to_datetime(df.index, format=format)
         return df
 
 
-class CSVBarDataLoader:
+class ParquetDataLoader:
     """
-    Provides a generic bar data CSV file loader.
-    """
-
-    @staticmethod
-    def load(file_path: PathLike[str] | str) -> pd.DataFrame:
-        """
-        Return the bar `pandas.DataFrame` loaded from the given CSV `file_path`.
-
-        Parameters
-        ----------
-        file_path : str, path object or file-like object
-            The path to the CSV file.
-
-        Returns
-        -------
-        pd.DataFrame
-
-        """
-        df = pd.read_csv(
-            file_path,
-            index_col="timestamp",
-            parse_dates=True,
-        )
-        df.index = pd.to_datetime(df.index, format="mixed")
-        return df
-
-
-class ParquetTickDataLoader:
-    """
-    Provides a generic tick data Parquet file loader.
+    Loads Parquet data to a `pandas.DataFrame`.
     """
 
     @staticmethod
     def load(
         file_path: PathLike[str] | str,
         timestamp_column: str = "timestamp",
+        format: str = "mixed",
     ) -> pd.DataFrame:
         """
-        Return the tick `pandas.DataFrame` loaded from the given Parquet `file_path`.
+        Return the `pandas.DataFrame` loaded from the given Parquet `file_path`.
 
         Parameters
         ----------
@@ -103,6 +75,8 @@ def load(
             The path to the Parquet file.
         timestamp_column: str
             Name of the timestamp column in the parquet data
+        format : str, default 'mixed'
+            The timestamp column format.
 
         Returns
         -------
@@ -111,31 +85,7 @@ def load(
         """
         df = pd.read_parquet(file_path)
         df = df.set_index(timestamp_column)
-        return df
-
-
-class ParquetBarDataLoader:
-    """
-    Provides a generic bar data Parquet file loader.
-    """
-
-    @staticmethod
-    def load(file_path: PathLike[str] | str) -> pd.DataFrame:
-        """
-        Return the bar `pandas.DataFrame` loaded from the given Parquet `file_path`.
-
-        Parameters
-        ----------
-        file_path : str, path object or file-like object
-            The path to the Parquet file.
-
-        Returns
-        -------
-        pd.DataFrame
-
-        """
-        df = pd.read_parquet(file_path)
-        df = df.set_index("timestamp")
+        df.index = pd.to_datetime(df.index, format=format)
         return df
 
 
diff --git a/nautilus_trader/persistence/wranglers_v2.py b/nautilus_trader/persistence/wranglers_v2.py
index 8847ee71a388..067938360d72 100644
--- a/nautilus_trader/persistence/wranglers_v2.py
+++ b/nautilus_trader/persistence/wranglers_v2.py
@@ -29,7 +29,6 @@
 from nautilus_trader.core.nautilus_pyo3 import QuoteTickDataWrangler as RustQuoteTickDataWrangler
 from nautilus_trader.core.nautilus_pyo3 import TradeTick as RustTradeTick
 from nautilus_trader.core.nautilus_pyo3 import TradeTickDataWrangler as RustTradeTickDataWrangler
-from nautilus_trader.model.data import BarType
 from nautilus_trader.model.instruments import Instrument
 
 
@@ -66,6 +65,13 @@ def decode(k, v):
             **{k.decode(): decode(k, v) for k, v in metadata.items() if k not in cls.IGNORE_KEYS},
         )
 
+    def scale_column(
+        self,
+        column: pd.Series,
+        dtype: pd.core.arrays.integer.IntegerDtype,
+    ) -> pd.Series:
+        return (column * 1e9).round().astype(dtype())
+
 
 class OrderBookDeltaDataWrangler(WranglerBase):
     """
@@ -140,10 +146,10 @@ def from_pandas(
         )
 
         # Scale prices and quantities
-        df["price"] = (df["price"] * 1e9).astype(pd.Int64Dtype())
-        df["size"] = (df["size"] * 1e9).round().astype(pd.UInt64Dtype())
+        df["price"] = super().scale_column(df["price"], pd.Int64Dtype)
+        df["size"] = super().scale_column(df["size"], pd.UInt64Dtype)
 
-        df["order_id"] = df["order_id"].astype(pd.UInt64Dtype())
+        df["order_id"] = super().scale_column(df["order_id"], pd.UInt64Dtype)
 
         # Process timestamps
         df["ts_event"] = (
@@ -248,17 +254,17 @@ def from_pandas(
         )
 
         # Scale prices and quantities
-        df["bid_price"] = (df["bid_price"] * 1e9).astype(pd.Int64Dtype())
-        df["ask_price"] = (df["ask_price"] * 1e9).astype(pd.Int64Dtype())
+        df["bid_price"] = super().scale_column(df["bid_price"], pd.Int64Dtype)
+        df["ask_price"] = super().scale_column(df["ask_price"], pd.Int64Dtype)
 
         # Create bid_size and ask_size columns
         if "bid_size" in df.columns:
-            df["bid_size"] = (df["bid_size"] * 1e9).astype(pd.Int64Dtype())
+            df["bid_size"] = super().scale_column(df["bid_size"], pd.Int64Dtype)
         else:
             df["bid_size"] = pd.Series([default_size * 1e9] * len(df), dtype=pd.UInt64Dtype())
 
         if "ask_size" in df.columns:
-            df["ask_size"] = (df["ask_size"] * 1e9).astype(pd.Int64Dtype())
+            df["ask_size"] = super().scale_column(df["ask_size"], pd.Int64Dtype)
         else:
             df["ask_size"] = pd.Series([default_size * 1e9] * len(df), dtype=pd.UInt64Dtype())
 
@@ -369,8 +375,8 @@ def from_pandas(
         )
 
         # Scale prices and quantities
-        df["price"] = (df["price"] * 1e9).astype(pd.Int64Dtype())
-        df["size"] = (df["size"] * 1e9).round().astype(pd.UInt64Dtype())
+        df["price"] = super().scale_column(df["price"], pd.Int64Dtype)
+        df["size"] = super().scale_column(df["size"], pd.UInt64Dtype)
 
         df["aggressor_side"] = df["aggressor_side"].map(_map_aggressor_side).astype(pd.UInt8Dtype())
         df["trade_id"] = df["trade_id"].astype(str)
@@ -413,8 +419,13 @@ class BarDataWrangler(WranglerBase):
 
     Parameters
     ----------
-    instrument : Instrument
-        The instrument for the data wrangler.
+    bar_type : str
+        The bar type for the data wrangler. For example,
+        "GBP/USD.SIM-1-MINUTE-BID-EXTERNAL"
+    price_precision: int
+        The price precision for the data wrangler.
+    size_precision: int
+        The size precision for the data wrangler.
 
     Warnings
     --------
@@ -425,7 +436,7 @@ class BarDataWrangler(WranglerBase):
 
     def __init__(
         self,
-        bar_type: BarType,
+        bar_type: str,
         price_precision: int,
         size_precision: int,
     ) -> None:
@@ -471,21 +482,29 @@ def from_pandas(
         Returns
         -------
         list[RustBar]
-            A list of PyO3 [pyclass] `TradeTick` objects.
+            A list of PyO3 [pyclass] `Bar` objects.
 
         """
         # Rename column
         df = df.rename(columns={"timestamp": "ts_event"})
 
-        # Scale prices and quantities
-        df["open"] = (df["open"] * 1e9).astype(pd.Int64Dtype())
-        df["high"] = (df["high"] * 1e9).astype(pd.Int64Dtype())
-        df["low"] = (df["low"] * 1e9).astype(pd.Int64Dtype())
-        df["clow"] = (df["close"] * 1e9).astype(pd.Int64Dtype())
+        # Check required columns
+        required_columns = {"open", "high", "low", "close", "ts_event"}
+        missing_columns = required_columns - set(df.columns)
+        if missing_columns:
+            raise ValueError(f"Missing columns: {missing_columns}")
+
+        # Scale OHLC
+        df["open"] = super().scale_column(df["open"], pd.Int64Dtype)
+        df["high"] = super().scale_column(df["high"], pd.Int64Dtype)
+        df["low"] = super().scale_column(df["low"], pd.Int64Dtype)
+        df["close"] = super().scale_column(df["close"], pd.Int64Dtype)
 
         if "volume" not in df.columns:
             df["volume"] = pd.Series([default_volume * 1e9] * len(df), dtype=pd.UInt64Dtype())
 
+        df["volume"] = super().scale_column(df["volume"], pd.UInt64Dtype)
+
         # Process timestamps
         df["ts_event"] = (
             pd.to_datetime(df["ts_event"], utc=True, format="mixed")
diff --git a/nautilus_trader/test_kit/providers.py b/nautilus_trader/test_kit/providers.py
index 50287b6a6074..fc2c29036f32 100644
--- a/nautilus_trader/test_kit/providers.py
+++ b/nautilus_trader/test_kit/providers.py
@@ -55,10 +55,8 @@
 from nautilus_trader.model.objects import Money
 from nautilus_trader.model.objects import Price
 from nautilus_trader.model.objects import Quantity
-from nautilus_trader.persistence.loaders import CSVBarDataLoader
-from nautilus_trader.persistence.loaders import CSVTickDataLoader
-from nautilus_trader.persistence.loaders import ParquetBarDataLoader
-from nautilus_trader.persistence.loaders import ParquetTickDataLoader
+from nautilus_trader.persistence.loaders import CSVDataLoader
+from nautilus_trader.persistence.loaders import ParquetDataLoader
 
 
 class TestInstrumentProvider:
@@ -600,22 +598,22 @@ def read_csv(self, path: str, **kwargs: Any) -> TextFileReader:
     def read_csv_ticks(self, path: str) -> pd.DataFrame:
         uri = self._make_uri(path=path)
         with fsspec.open(uri) as f:
-            return CSVTickDataLoader.load(file_path=f)
+            return CSVDataLoader.load(file_path=f)
 
     def read_csv_bars(self, path: str) -> pd.DataFrame:
         uri = self._make_uri(path=path)
         with fsspec.open(uri) as f:
-            return CSVBarDataLoader.load(file_path=f)
+            return CSVDataLoader.load(file_path=f)
 
     def read_parquet_ticks(self, path: str, timestamp_column: str = "timestamp") -> pd.DataFrame:
         uri = self._make_uri(path=path)
         with fsspec.open(uri) as f:
-            return ParquetTickDataLoader.load(file_path=f, timestamp_column=timestamp_column)
+            return ParquetDataLoader.load(file_path=f, timestamp_column=timestamp_column)
 
     def read_parquet_bars(self, path: str) -> pd.DataFrame:
         uri = self._make_uri(path=path)
         with fsspec.open(uri) as f:
-            return ParquetBarDataLoader.load(file_path=f)
+            return ParquetDataLoader.load(file_path=f)
 
 
 class TestDataGenerator:

From 3c8ec14c7123388983e5cc0b3bbbaca234852cad Mon Sep 17 00:00:00 2001
From: Ben Singleton <benfsingleton@gmail.com>
Date: Fri, 10 Nov 2023 22:37:49 -0500
Subject: [PATCH 2/6] update docs and add gateway config

---
 docs/integrations/ib.md                       | 268 ++++++++----------
 .../adapters/interactive_brokers/config.py    |   6 +-
 .../adapters/interactive_brokers/gateway.py   |  56 ++--
 3 files changed, 160 insertions(+), 170 deletions(-)

diff --git a/docs/integrations/ib.md b/docs/integrations/ib.md
index 282c0206b0b3..149a4a52d54b 100644
--- a/docs/integrations/ib.md
+++ b/docs/integrations/ib.md
@@ -1,38 +1,49 @@
 # Interactive Brokers
 
-Interactive Brokers (IB) is a trading platform where you can trade stocks, options, futures, currencies, bonds, funds, and crypto. NautilusTrader provides an adapter to integrate with IB using their [Trader Workstation (TWS) API](https://interactivebrokers.github.io/tws-api/index.html) via their Python library, [ibapi](https://github.com/nautechsystems/ibapi).
+Interactive Brokers (IB) is a trading platform that allows trading in a wide range of financial instruments, including stocks, options, futures, currencies, bonds, funds, and cryptocurrencies. NautilusTrader offers an adapter to integrate with IB using their [Trader Workstation (TWS) API](https://interactivebrokers.github.io/tws-api/index.html) through their Python library, [ibapi](https://github.com/nautechsystems/ibapi).
 
-The TWS API is an interface to IB's standalone trading applications, TWS and IB Gateway, which can be downloaded on IB's website. If you have not already installed TWS or IB Gateway, follow Interactive Brokers' [Initial Setup](https://interactivebrokers.github.io/tws-api/initial_setup.html) guide. You will define a connection to either of these two applications in NautilusTrader's `InteractiveBrokersClient`. 
+The TWS API serves as an interface to IB's standalone trading applications: TWS and IB Gateway. Both can be downloaded from the IB website. If you haven't installed TWS or IB Gateway yet, refer to the [Initial Setup](https://interactivebrokers.github.io/tws-api/initial_setup.html) guide. In NautilusTrader, you'll establish a connection to one of these applications via the `InteractiveBrokersClient`.
 
-Another (and perhaps easier way) to get started is to use a [dockerized version](https://github.com/unusualalpha/ib-gateway-docker/pkgs/container/ib-gateway) of IB Gateway, which is also what you would use when deploying your trading strategies on a hosted cloud platform. You will need [Docker](https://www.docker.com/) installed on your machine and the [docker](https://pypi.org/project/docker/) Python package, which is conveniently bundled with NautilusTrader as an extra package.
+Alternatively, you can start with a [dockerized version](https://github.com/unusualalpha/ib-gateway-docker/pkgs/container/ib-gateway) of the IB Gateway, particularly useful when deploying trading strategies on a hosted cloud platform. This requires having [Docker](https://www.docker.com/) installed on your machine, along with the [docker](https://pypi.org/project/docker/) Python package, which NautilusTrader conveniently includes as an extra package.
 
-**Note**: The standalone TWS and IB Gateway applications require human intervention to specify a username, password and trading mode (live or paper trading) at startup. The dockerized IB Gateway is able to do so programmatically.
+**Note**: The standalone TWS and IB Gateway applications necessitate manual input of username, password, and trading mode (live or paper) at startup. The dockerized version of the IB Gateway handles these steps programmatically.
 
 ## Installation
 
-To install the latest nautilus-trader package with the `ibapi` and optional `docker` extra dependencies using pip, run:
+To install the latest nautilus-trader package along with the `ibapi` and optional `docker` dependencies using pip, execute:
 
 ```
 pip install -U "nautilus_trader[ib,docker]"
 ```
 
-To install using poetry, run:
+For installation via poetry, use:
 
 ```
 poetry add "nautilus_trader[ib,docker]"
 ```
 
-**Note**: IB does not provide wheels for `ibapi`, so Nautilus [repackages]( https://pypi.org/project/nautilus-ibapi/) and releases it to PyPI.
+**Note**: Because IB does not provide wheels for `ibapi`, NautilusTrader [repackages]( https://pypi.org/project/nautilus-ibapi/) it for release on PyPI.
+
 
 ## Getting Started
 
-Before writing strategies, TWS / IB Gateway needs to be running. Launch either of the two standalone applications and enter your credentials, or start the dockerized IB Gateway using `InteractiveBrokersGateway` (make sure Docker is running in the background already!):
+Before deploying strategies, ensure that TWS / IB Gateway is running. Launch one of the standalone applications and log in with your credentials, or start the dockerized IB Gateway using `InteractiveBrokersGateway`:
 
-```
+```python
 from nautilus_trader.adapters.interactive_brokers.gateway import InteractiveBrokersGateway
 
+
+gateway_config = InteractiveBrokersGatewayConfig(
+    username="test",
+    password="test",
+    trading_mode="paper",
+    start=True
+)
+
 # This may take a short while to start up, especially the first time
-gateway = InteractiveBrokersGateway(username="test", password="test", start=True)
+gateway = InteractiveBrokersGateway(
+    config=gateway_config
+)
 
 # Confirm you are logged in
 print(gateway.is_logged_in(gateway.container))
@@ -41,31 +52,29 @@ print(gateway.is_logged_in(gateway.container))
 print(gateway.container.logs())
 ```
 
-**Note**: There are two options for supplying your credentials to the Interactive Brokers Gateway, Exec and Data clients.
-Either pass the corresponding `username` and `password` values to the config dictionaries, or
-set the following environment variables: 
+**Note**: To supply credentials to the Interactive Brokers Gateway, either pass the `username` and `password` to the config dictionaries, or set the following environment variables:
 - `TWS_USERNAME`
 - `TWS_PASSWORD`
 
 ## Overview
 
-The adapter is comprised of the following major components:
-- `InteractiveBrokersClient` which uses `ibapi` to execute TWS API requests, supporting all other integration classes.
-- `HistoricInteractiveBrokersClient` which provides a straightforward way to retrieve instruments and historical bar and tick data to load into the catalog (generally for backtesting).
-- `InteractiveBrokersInstrumentProvider` which retrieves or queries instruments for trading.
-- `InteractiveBrokersDataClient` which connects to the `Gateway` and streams market data for trading.
-- `InteractiveBrokersExecutionClient` which retrieves account information and executes orders for trading.
+The adapter includes several major components:
+- `InteractiveBrokersClient`: Executes TWS API requests using `ibapi`.
+- `HistoricInteractiveBrokersClient`: Provides methods for retrieving instruments and historical data, useful for backtesting.
+- `InteractiveBrokersInstrumentProvider`: Retrieves or queries instruments for trading.
+- `InteractiveBrokersDataClient`: Connects to the Gateway for streaming market data.
+- `InteractiveBrokersExecutionClient`: Handles account information and executes trades.
 
 ## Instruments & Contracts
 
-In Interactive Brokers, the concept of a NautilusTrader `Instrument` is called a [Contract](https://interactivebrokers.github.io/tws-api/contracts.html). A Contract can be represented in two ways: a [basic contract](https://interactivebrokers.github.io/tws-api/classIBApi_1_1Contract.html) or a [detailed contract](https://interactivebrokers.github.io/tws-api/classIBApi_1_1ContractDetails.html), which are defined in the adapter using the classes `IBContract` and `IBContractDetails`, respectively. Contract details include important information like supported order types and trading hours, which aren't found in the basic contract. As a result, `IBContractDetails` can be converted to an `Instrument` but `IBContract` cannot.
+In IB, a NautilusTrader `Instrument` is equivalent to a Contract](https://interactivebrokers.github.io/tws-api/contracts.html). Contracts can be either a [basic contract](https://interactivebrokers.github.io/tws-api/classIBApi_1_1Contract.html) or a more [detailed](https://interactivebrokers.github.io/tws-api/classIBApi_1_1ContractDetails.html) version (ContractDetails). The adapter models these using `IBContract` and `IBContractDetails` classes. The latter includes critical data like order types and trading hours, which are absent in the basic contract, making `IBContractDetails` convertible to an `Instrument`, unlike IBContract.
 
-To find basic contract information, use the [IB Contract Information Center](https://pennies.interactivebrokers.com/cstools/contract_info/). 
+To search for contract information, use the [IB Contract Information Center](https://pennies.interactivebrokers.com/cstools/contract_info/).
 
 Examples of `IBContracts`:
 ```
 from nautilus_trader.adapters.interactive_brokers.common import IBContract
- 
+
 # Stock
 IBContract(secType='STK', exchange='SMART', primaryExchange='ARCA', symbol='SPY')
 
@@ -90,13 +99,16 @@ IBContract(secType='CRYPTO', symbol='ETH', exchange='PAXOS', currency='USD')
 
 ## Historical Data & Backtesting
 
-The first step in developing strategies using the IB adapter typically involves fetching historical data that can be used for backtesting. The `HistoricInteractiveBrokersClient` provides a number of convenience methods to request data and save it to the catalog.
+When developing strategies with the IB adapter, the first step usually involves acquiring historical data for backtesting. The `HistoricInteractiveBrokersClient` offers methods to request and save this data.
 
-The following example illustrates retrieving and saving instrument, bar, quote tick, and trade tick data. Read more about using the data for backtesting [here].
+Here's an example of retrieving and saving instrument and bar data. A more comprehensive example is available [here](https://github.com/nautechsystems/nautilus_trader/blob/master/examples/live/interactive_brokers/historic_download.py).
+
+```python
+import datetime
+
+from nautilus_trader.adapters.interactive_brokers.historic import HistoricInteractiveBrokersClient
+from nautilus_trader.persistence.catalog import ParquetDataCatalog
 
-```
-# Define path to existing catalog or desired new catalog path
-CATALOG_PATH = "./catalog"
 
 async def main():
     contract = IBContract(
@@ -105,52 +117,42 @@ async def main():
         exchange="SMART",
         primaryExchange="NASDAQ",
     )
-    client = HistoricInteractiveBrokersClient(
-        catalog=ParquetDataCatalog(CATALOG_PATH)
-    )
-    # By default, data is written to the catalog
-    await client.get_instruments(contract=contract)
-
-    # All the methods also return the retrieved objects
-    # so you can introspect them or explicitly write them
-    # to the catalog
-    bars = await client.get_historical_bars(
-        contract=contract,
-        bar_type=...
-        write_to_catalog=False
+    client = HistoricInteractiveBrokersClient()
+
+    instruments = await client.request_instruments(
+        contracts=[contract],
     )
-    print(bars)
-    client.catalog.write_data(bars)
-    
-    await client.get_historical_ticks(
-        contract=contract,
-        bar_type..
+
+    bars = await client.request_bars(
+        bar_specifications=["1-HOUR-LAST", "30-MINUTE-MID"],
+        end_date_time=datetime.datetime(2023, 11, 6, 16, 30),
+        tz_name="America/New_York",
+        duration="1 D",
+        contracts=[contract],
     )
 
-if __name__ == "__main__":
-    asyncio.run(main())
+    catalog = ParquetDataCatalog("./catalog")
+    catalog.write_data(instruments)
+    catalog.write_data(bars)
 ```
 
 ## Live Trading
 
-To live trade or paper trade, a `TradingNode` needs to be built and run with a `InteractiveBrokersDataClient` and a `InteractiveBrokersExecutionClient`, which both rely on a `InteractiveBrokersInstrumentProvider`.
+Engaging in live or paper trading requires constructing and running a `TradingNode`. This node incorporates both `InteractiveBrokersDataClient` and `InteractiveBrokersExecutionClient`, which depend on the `InteractiveBrokersInstrumentProvider` to operate.
 
 ### InstrumentProvider
 
-To retrieve instruments, you will need to use the `InteractiveBrokersInstrumentProvider`. This provider is also used under the hood in the `HistoricInteractiveBrokersClient` to retrieve instruments for data collection.
+The `InteractiveBrokersInstrumentProvider` class functions as a bridge for accessing financial instrument data from IB. Configurable through `InteractiveBrokersInstrumentProviderConfig`, it allows for the customization of various instrument type parameters. Additionally, this provider offers specialized methods to build and retrieve the entire futures and options chains for given underlying securities.
 
 ```
-from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersDataClientConfig
+from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersInstrumentProviderConfig
 
-gateway_config = InteractiveBrokersGatewayConfig(username="test", password="test")
 
-# Specify instruments to retrieve in load_ids and / or load_contracts
-# The following parameters should only be specified if retrieving futures or options: build_futures_chain, build_options_chain, min_expiry_days, max_expiry_days
-instrument_provider = InteractiveBrokersInstrumentProviderConfig(
-    build_futures_chain=False,  # optional, only if fetching futures
-    build_options_chain=False,  # optional, only if fetching futures
-    min_expiry_days=10,  # optional, only if fetching futures / options
-    max_expiry_days=60,  # optional, only if fetching futures / options
+instrument_provider_config = InteractiveBrokersInstrumentProviderConfig(
+    build_futures_chain=False,  # Set to True if fetching futures
+    build_options_chain=False,  # Set to True if fetching options
+    min_expiry_days=10,         # Relevant for futures/options with expiration
+    max_expiry_days=60,         # Relevant for futures/options with expiration
     load_ids=frozenset(
         [
             "EUR/USD.IDEALPRO",
@@ -162,121 +164,97 @@ instrument_provider = InteractiveBrokersInstrumentProviderConfig(
             "ESZ27.CME",
         ],
     ),
-    load_contracts=frozenset(ib_contracts),
-)
-
-# Configure the trading node
-config_node = TradingNodeConfig(
-    trader_id="IB-001",
-    logging=LoggingConfig(log_level="INFO"),
-    data_clients={
-        "IB": InteractiveBrokersDataClientConfig(
-            ibg_host="127.0.0.1",
-            ibg_port=4002,
-            ibg_client_id=1,
-            handle_revised_bars=False,
-            use_regular_trading_hours=True,
-            market_data_type=IBMarketDataTypeEnum.DELAYED_FROZEN,  # https://interactivebrokers.github.io/tws-api/market_data_type.html
-            instrument_provider=instrument_provider,
-            gateway=gateway,
-        ),
-    },
-    timeout_connection=90.0,
+    load_contracts=frozenset(
+        [
+            IBContract(secType='STK', symbol='SPY', exchange='SMART', primaryExchange='ARCA'),
+            IBContract(secType='STK', symbol='AAPL', exchange='SMART', primaryExchange='NASDAQ')
+        ]
+    ),
 )
-
-node = TradingNode(config=config_node)
-
-node.trader.add_actor(downloader)
-
-# Register your client factories with the node (can take user defined factories)
-node.add_data_client_factory("InteractiveBrokers", InteractiveBrokersLiveDataClientFactory)
-node.add_exec_client_factory("InteractiveBrokers", InteractiveBrokersLiveExecClientFactory)
-node.build()
-
-# Stop and dispose of the node with SIGINT/CTRL+C
-if __name__ == "__main__":
-    try:
-        node.run()
-    finally:
-        node.dispose()
-
 ```
 
-Interactive Brokers allows searching for instruments via the `reqMatchingSymbols` API ([docs](https://interactivebrokers.github.io/tws-api/matching_symbols.html)), which, if given enough information
-can usually resolve a filter into an actual contract(s). A node can request instruments to be loaded by passing 
-configuration to the `InstrumentProviderConfig` when initialising a `TradingNodeConfig` (note that while `filters`
-is a dict, it must be converted to a tuple when passed to `InstrumentProviderConfig`).
+### Data Client
 
-At a minimum, you must specify the `secType` (security type) and `symbol` (equities etc) or `pair` (FX). See examples 
-queries below for common use cases 
+`InteractiveBrokersDataClient` interfaces with Interactive Brokers for streaming and retrieving market data. Upon connection, it configures the [market data type](https://interactivebrokers.github.io/tws-api/market_data_type.html) and loads instruments based on the settings in `InteractiveBrokersInstrumentProviderConfig`. This client can subscribe to and unsubscribe from various market data types, including quote ticks, trade ticks, and bars.
 
-Example config: 
+Configurable through `InteractiveBrokersDataClientConfig`, it allows adjustments for handling revised bars, trading hours preferences, and market data types (e.g., `IBMarketDataTypeEnum.REALTIME` or `IBMarketDataTypeEnum.DELAYED_FROZEN`).
 
 ```python
-from nautilus_trader.adapters.interactive_brokers.common import IBContract
+from nautilus_trader.adapters.interactive_brokers.config import IBMarketDataTypeEnum
 from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersDataClientConfig
-from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersInstrumentProviderConfig
-from nautilus_trader.config import TradingNodeConfig
 
-config_node = TradingNodeConfig(
-    data_clients={
-        "IB": InteractiveBrokersDataClientConfig(
-            instrument_provider=InteractiveBrokersInstrumentProviderConfig(
-                load_ids={"EUR/USD.IDEALPRO", "AAPL.NASDAQ"},
-                load_contracts={IBContract(secType="CONTFUT", exchange="CME", symbol="MES")},
-            )
-    ),
-    ...
+
+data_client_config = InteractiveBrokersDataClientConfig(
+    ibg_port=4002,
+    handle_revised_bars=False,
+    use_regular_trading_hours=True,
+    market_data_type=IBMarketDataTypeEnum.DELAYED_FROZEN,  # Default is REALTIME if not set
+    instrument_provider=instrument_provider_config,
+    gateway=gateway_config,
 )
 ```
 
-### Data Client
-- `InteractiveBrokersDataClient` which streams market data.
-
 ### Execution Client
-- `InteractiveBrokersExecutionClient` which retrieves account information and executes orders.
 
-### Full Configuration
-The most common use case is to configure a live `TradingNode` to include Interactive Brokers
-data and execution clients. To achieve this, add an `IB` section to your client
-configuration(s) and set the environment variables to your TWS (Traders Workstation) credentials:
+The `InteractiveBrokersExecutionClient` facilitates executing trades, accessing account information, and processing order and trade-related details. It encompasses a range of methods for order management, including reporting order statuses, placing new orders, and modifying or canceling existing ones. Additionally, it generates position reports, although trade reports are not yet implemented.
 
 ```python
-import os
 from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersExecClientConfig
-from nautilus_trader.config import TradingNodeConfig
+from nautilus_trader.config import RoutingConfig
 
 
-config = TradingNodeConfig(
-    data_clients={
-        "IB": InteractiveBrokersDataClientConfig(
-            username=os.getenv("TWS_USERNAME"),
-            password=os.getenv("TWS_PASSWORD"),
-            ...  # Omitted
-    },
-    exec_clients = {
-        "IB": InteractiveBrokersExecClientConfig(
-            username=os.getenv("TWS_USERNAME"),
-            password=os.getenv("TWS_PASSWORD"),
-            ...  # Omitted
-    },
-    ...  # Omitted
+exec_client_config = InteractiveBrokersExecClientConfig(
+    ibg_port=4002,
+    account_id="DU123456",  # Must match the connected IB Gateway/TWS
+    gateway=gateway_config,
+    instrument_provider=instrument_provider_config,
+    routing=RoutingConfig(
+        default=True,
+    )
 )
 ```
 
-Then, create a `TradingNode` and add the client factories:
+### Full Configuration
+
+Setting up a complete trading environment typically involves configuring a `TradingNodeConfig`, which includes data and execution client configurations. Additional configurations are specified in `LiveDataEngineConfig` to accommodate IB-specific requirements. A `TradingNode` is then instantiated from these configurations, and factories for creating `InteractiveBrokersDataClient` and `InteractiveBrokersExecutionClient` are added. Finally, the node is built and run.
+
+For a comprehensive example, refer to this [guide](https://github.com/nautechsystems/nautilus_trader/blob/master/examples/live/interactive_brokers/interactive_brokers_example.py).
+
 
 ```python
+from nautilus_trader.adapters.interactive_brokers.common import IB_VENUE
 from nautilus_trader.adapters.interactive_brokers.factories import InteractiveBrokersLiveDataClientFactory
 from nautilus_trader.adapters.interactive_brokers.factories import InteractiveBrokersLiveExecClientFactory
+from nautilus_trader.config import LiveDataEngineConfig
+from nautilus_trader.config import LoggingConfig
+from nautilus_trader.config import TradingNodeConfig
+from nautilus_trader.live.node import TradingNode
+
 
-# Instantiate the live trading node with a configuration
-node = TradingNode(config=config)
+# ... [continuing from prior example code] ...
+
+# Configure the trading node
+config_node = TradingNodeConfig(
+    trader_id="TESTER-001",
+    logging=LoggingConfig(log_level="INFO"),
+    data_clients={"IB": data_client_config},
+    exec_clients={"IB": exec_client_config},
+    data_engine=LiveDataEngineConfig(
+        time_bars_timestamp_on_close=False,  # Use opening time as `ts_event`, as per IB standard
+        validate_data_sequence=True,         # Discards bars received out of sequence
+    ),
+)
 
-# Register the client factories with the node
+node = TradingNode(config=config_node)
 node.add_data_client_factory("IB", InteractiveBrokersLiveDataClientFactory)
 node.add_exec_client_factory("IB", InteractiveBrokersLiveExecClientFactory)
-
-# Finally build the node
 node.build()
+node.portfolio.set_specific_venue(IB_VENUE)
+
+if __name__ == "__main__":
+    try:
+        node.run()
+    finally:
+        # Stop and dispose of the node with SIGINT/CTRL+C
+        node.dispose()
 ```
diff --git a/nautilus_trader/adapters/interactive_brokers/config.py b/nautilus_trader/adapters/interactive_brokers/config.py
index 43f762479335..8bd4692c62bf 100644
--- a/nautilus_trader/adapters/interactive_brokers/config.py
+++ b/nautilus_trader/adapters/interactive_brokers/config.py
@@ -49,6 +49,8 @@ class InteractiveBrokersGatewayConfig(NautilusConfig, frozen=True):
 
     username: str | None = None
     password: str | None = None
+    host: str | None = "localhost"
+    port: Literal[4001, 4002] | None = None
     trading_mode: Literal["paper", "live"] = "paper"
     start: bool = False
     read_only_api: bool = True
@@ -155,8 +157,8 @@ class InteractiveBrokersExecClientConfig(LiveExecClientConfig, frozen=True):
     ----------
     ibg_host : str, default "127.0.0.1"
         The hostname or ip address for the IB Gateway or TWS.
-    ibg_port : int, default for "paper" 4002, or "live" 4001
-        The port for the gateway server.
+    ibg_port : int
+        The port for the gateway server ("paper" 4002, or "live" 4001).
     ibg_client_id: int, default 1
         The client_id to be passed into connect call.
     ibg_account_id : str
diff --git a/nautilus_trader/adapters/interactive_brokers/gateway.py b/nautilus_trader/adapters/interactive_brokers/gateway.py
index 074a2bbe10a9..c808323d45da 100644
--- a/nautilus_trader/adapters/interactive_brokers/gateway.py
+++ b/nautilus_trader/adapters/interactive_brokers/gateway.py
@@ -15,19 +15,17 @@
 
 import logging
 import os
-import warnings
 from enum import IntEnum
 from time import sleep
-from typing import ClassVar
+from typing import ClassVar, Literal
+
+from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersGatewayConfig
 
 
 try:
     import docker
 except ImportError as e:
-    warnings.warn(
-        f"Docker required for Gateway, install manually via `pip install docker` ({e})",
-    )
-    docker = None
+    raise RuntimeError("Docker required for Gateway, install via `pip install docker`") from e
 
 
 class ContainerStatus(IntEnum):
@@ -55,24 +53,34 @@ def __init__(
         password: str | None = None,
         host: str | None = "localhost",
         port: int | None = None,
-        trading_mode: str | None = "paper",
+        trading_mode: Literal["paper", "live"] | None = "paper",
         start: bool = False,
         read_only_api: bool = True,
         timeout: int = 90,
         logger: logging.Logger | None = None,
+        config: InteractiveBrokersGatewayConfig | None = None,
     ):
-        username = username if username is not None else os.environ["TWS_USERNAME"]
-        password = password if password is not None else os.environ["TWS_PASSWORD"]
-        assert username is not None, "`username` not set nor available in env `TWS_USERNAME`"
-        assert password is not None, "`password` not set nor available in env `TWS_PASSWORD`"
-        self.username = username
-        self.password = password
+        if config:
+            username = config.username
+            password = config.password
+            host = config.host
+            port = config.port
+            trading_mode = config.trading_mode
+            start = config.start
+            read_only_api = config.read_only_api
+            timeout = config.timeout
+
+        self.username = username or os.getenv("TWS_USERNAME")
+        self.password = password or os.getenv("TWS_PASSWORD")
+        if username is None:
+            raise ValueError("`username` not set nor available in env `TWS_USERNAME`")
+        if password is None:
+            raise ValueError("`password` not set nor available in env `TWS_PASSWORD`")
+
         self.trading_mode = trading_mode
         self.read_only_api = read_only_api
         self.host = host
         self.port = port or self.PORTS[trading_mode]
-        if docker is None:
-            raise RuntimeError("Docker not installed")
         self._docker = docker.from_env()
         self._container = None
         self.log = logger or logging.getLogger("nautilus_trader")
@@ -165,11 +173,10 @@ def start(self, wait: int | None = 90):
             for _ in range(wait):
                 if self.is_logged_in(container=self._container):
                     break
-                else:
-                    self.log.debug("Waiting for IB Gateway to start ..")
-                    sleep(1)
+                self.log.debug("Waiting for IB Gateway to start ..")
+                sleep(1)
             else:
-                raise GatewayLoginFailure
+                raise RuntimeError(f"Gateway `{self.CONTAINER_NAME}-{self.port}` not ready")
 
         self.log.info(
             f"Gateway `{self.CONTAINER_NAME}-{self.port}` ready. VNC port is {self.port+100}",
@@ -178,8 +185,8 @@ def start(self, wait: int | None = 90):
     def safe_start(self, wait: int = 90):
         try:
             self.start(wait=wait)
-        except ContainerExists:
-            return
+        except docker.errors.APIError as e:
+            raise RuntimeError("Container already exists") from e
 
     def stop(self):
         if self.container:
@@ -189,8 +196,11 @@ def stop(self):
     def __enter__(self):
         self.start()
 
-    def __exit__(self, type, value, traceback):
-        self.stop()
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        try:
+            self.stop()
+        except Exception as e:
+            logging.error("Error stopping container: %s", e)
 
 
 # -- Exceptions -----------------------------------------------------------------------------------

From 12fc140621b3682bc1d95ada4109a22f1037b774 Mon Sep 17 00:00:00 2001
From: Ben Singleton <benfsingleton@gmail.com>
Date: Fri, 10 Nov 2023 22:48:23 -0500
Subject: [PATCH 3/6] undo unrelated changes

---
 docs/tutorials/backtest_high_level.md         |  6 +-
 .../notebooks/external_data_backtest.ipynb    |  4 +-
 nautilus_trader/persistence/loaders.py        | 78 +++++++++++++++----
 nautilus_trader/test_kit/providers.py         | 14 ++--
 4 files changed, 77 insertions(+), 25 deletions(-)

diff --git a/docs/tutorials/backtest_high_level.md b/docs/tutorials/backtest_high_level.md
index 29e6baa5b20e..b4454f4ca45b 100644
--- a/docs/tutorials/backtest_high_level.md
+++ b/docs/tutorials/backtest_high_level.md
@@ -29,7 +29,7 @@ from nautilus_trader.backtest.node import BacktestNode, BacktestVenueConfig, Bac
 from nautilus_trader.config.common import ImportableStrategyConfig
 from nautilus_trader.persistence.catalog import ParquetDataCatalog
 from nautilus_trader.persistence.wranglers import QuoteTickDataWrangler
-from nautilus_trader.persistence.loaders import CSVDataLoader
+from nautilus_trader.test_kit.providers import CSVTickDataLoader
 from nautilus_trader.test_kit.providers import TestInstrumentProvider
 ```
 
@@ -64,7 +64,7 @@ Then we can create Nautilus `QuoteTick` objects by processing the DataFrame with
 
 ```python
 # Here we just take the first data file found and load into a pandas DataFrame
-df = CSVDataLoader.load(raw_files[0], timestamp_column=0, format="%Y%m%d %H%M%S%f")
+df = CSVTickDataLoader.load(raw_files[0], index_col=0, format="%Y%m%d %H%M%S%f")
 df.columns = ["bid_price", "ask_price"]
 
 # Process quote ticks using a wrangler
@@ -178,4 +178,4 @@ node = BacktestNode(configs=[config])
 
 results = node.run()
 results
-```
+```
\ No newline at end of file
diff --git a/examples/notebooks/external_data_backtest.ipynb b/examples/notebooks/external_data_backtest.ipynb
index e3b1f9d73161..c3a26dffa2ff 100644
--- a/examples/notebooks/external_data_backtest.ipynb
+++ b/examples/notebooks/external_data_backtest.ipynb
@@ -38,7 +38,7 @@
     "from nautilus_trader.config.common import ImportableStrategyConfig\n",
     "from nautilus_trader.persistence.catalog import ParquetDataCatalog\n",
     "from nautilus_trader.persistence.wranglers import QuoteTickDataWrangler\n",
-    "from nautilus_trader.test_kit.providers import CSVDataLoader\n",
+    "from nautilus_trader.test_kit.providers import CSVTickDataLoader\n",
     "from nautilus_trader.test_kit.providers import TestInstrumentProvider"
    ]
   },
@@ -73,7 +73,7 @@
    "outputs": [],
    "source": [
     "# Here we just take the first data file found and load into a pandas DataFrame\n",
-    "df = CSVDataLoader.load(raw_files[0], index_col=0, format=\"%Y%m%d %H%M%S%f\")\n",
+    "df = CSVTickDataLoader.load(raw_files[0], index_col=0, format=\"%Y%m%d %H%M%S%f\")\n",
     "df.columns = [\"bid_price\", \"ask_price\"]\n",
     "\n",
     "# Process quote ticks using a wrangler\n",
diff --git a/nautilus_trader/persistence/loaders.py b/nautilus_trader/persistence/loaders.py
index f8ee0451cbce..06dfd4f05015 100644
--- a/nautilus_trader/persistence/loaders.py
+++ b/nautilus_trader/persistence/loaders.py
@@ -18,26 +18,26 @@
 import pandas as pd
 
 
-class CSVDataLoader:
+class CSVTickDataLoader:
     """
-    Loads a CSV file to a `pandas.DataFrame`.
+    Provides a generic tick data CSV file loader.
     """
 
     @staticmethod
     def load(
         file_path: PathLike[str] | str,
-        timestamp_column: str | int = "timestamp",
+        index_col: str | int = "timestamp",
         format: str = "mixed",
     ) -> pd.DataFrame:
         """
-        Return a `pandas.DataFrame` loaded from the given CSV `file_path`.
+        Return a tick `pandas.DataFrame` loaded from the given CSV `file_path`.
 
         Parameters
         ----------
         file_path : str, path object or file-like object
             The path to the CSV file.
-        timestamp_column : str | int, default 'timestamp'
-            Name of the timestamp column in the CSV file
+        index_col : str | int, default 'timestamp'
+            The index column.
         format : str, default 'mixed'
             The timestamp column format.
 
@@ -48,26 +48,54 @@ def load(
         """
         df = pd.read_csv(
             file_path,
-            index_col=timestamp_column,
+            index_col=index_col,
             parse_dates=True,
         )
         df.index = pd.to_datetime(df.index, format=format)
         return df
 
 
-class ParquetDataLoader:
+class CSVBarDataLoader:
     """
-    Loads Parquet data to a `pandas.DataFrame`.
+    Provides a generic bar data CSV file loader.
+    """
+
+    @staticmethod
+    def load(file_path: PathLike[str] | str) -> pd.DataFrame:
+        """
+        Return the bar `pandas.DataFrame` loaded from the given CSV `file_path`.
+
+        Parameters
+        ----------
+        file_path : str, path object or file-like object
+            The path to the CSV file.
+
+        Returns
+        -------
+        pd.DataFrame
+
+        """
+        df = pd.read_csv(
+            file_path,
+            index_col="timestamp",
+            parse_dates=True,
+        )
+        df.index = pd.to_datetime(df.index, format="mixed")
+        return df
+
+
+class ParquetTickDataLoader:
+    """
+    Provides a generic tick data Parquet file loader.
     """
 
     @staticmethod
     def load(
         file_path: PathLike[str] | str,
         timestamp_column: str = "timestamp",
-        format: str = "mixed",
     ) -> pd.DataFrame:
         """
-        Return the `pandas.DataFrame` loaded from the given Parquet `file_path`.
+        Return the tick `pandas.DataFrame` loaded from the given Parquet `file_path`.
 
         Parameters
         ----------
@@ -75,8 +103,6 @@ def load(
             The path to the Parquet file.
         timestamp_column: str
             Name of the timestamp column in the parquet data
-        format : str, default 'mixed'
-            The timestamp column format.
 
         Returns
         -------
@@ -85,7 +111,31 @@ def load(
         """
         df = pd.read_parquet(file_path)
         df = df.set_index(timestamp_column)
-        df.index = pd.to_datetime(df.index, format=format)
+        return df
+
+
+class ParquetBarDataLoader:
+    """
+    Provides a generic bar data Parquet file loader.
+    """
+
+    @staticmethod
+    def load(file_path: PathLike[str] | str) -> pd.DataFrame:
+        """
+        Return the bar `pandas.DataFrame` loaded from the given Parquet `file_path`.
+
+        Parameters
+        ----------
+        file_path : str, path object or file-like object
+            The path to the Parquet file.
+
+        Returns
+        -------
+        pd.DataFrame
+
+        """
+        df = pd.read_parquet(file_path)
+        df = df.set_index("timestamp")
         return df
 
 
diff --git a/nautilus_trader/test_kit/providers.py b/nautilus_trader/test_kit/providers.py
index 32353389850c..523d67959bd7 100644
--- a/nautilus_trader/test_kit/providers.py
+++ b/nautilus_trader/test_kit/providers.py
@@ -56,8 +56,10 @@
 from nautilus_trader.model.objects import Money
 from nautilus_trader.model.objects import Price
 from nautilus_trader.model.objects import Quantity
-from nautilus_trader.persistence.loaders import CSVDataLoader
-from nautilus_trader.persistence.loaders import ParquetDataLoader
+from nautilus_trader.persistence.loaders import CSVBarDataLoader
+from nautilus_trader.persistence.loaders import CSVTickDataLoader
+from nautilus_trader.persistence.loaders import ParquetBarDataLoader
+from nautilus_trader.persistence.loaders import ParquetTickDataLoader
 
 
 class TestInstrumentProvider:
@@ -609,22 +611,22 @@ def read_csv(self, path: str, **kwargs: Any) -> TextFileReader:
     def read_csv_ticks(self, path: str) -> pd.DataFrame:
         uri = self._make_uri(path=path)
         with fsspec.open(uri) as f:
-            return CSVDataLoader.load(file_path=f)
+            return CSVTickDataLoader.load(file_path=f)
 
     def read_csv_bars(self, path: str) -> pd.DataFrame:
         uri = self._make_uri(path=path)
         with fsspec.open(uri) as f:
-            return CSVDataLoader.load(file_path=f)
+            return CSVBarDataLoader.load(file_path=f)
 
     def read_parquet_ticks(self, path: str, timestamp_column: str = "timestamp") -> pd.DataFrame:
         uri = self._make_uri(path=path)
         with fsspec.open(uri) as f:
-            return ParquetDataLoader.load(file_path=f, timestamp_column=timestamp_column)
+            return ParquetTickDataLoader.load(file_path=f, timestamp_column=timestamp_column)
 
     def read_parquet_bars(self, path: str) -> pd.DataFrame:
         uri = self._make_uri(path=path)
         with fsspec.open(uri) as f:
-            return ParquetDataLoader.load(file_path=f)
+            return ParquetBarDataLoader.load(file_path=f)
 
 
 class TestDataGenerator:

From 234a59affaaa547dac9d1e149537468d5bcd03ea Mon Sep 17 00:00:00 2001
From: Ben Singleton <benfsingleton@gmail.com>
Date: Fri, 10 Nov 2023 23:03:05 -0500
Subject: [PATCH 4/6] final fixes

---
 examples/notebooks/backtest_example.ipynb     |  8 ---
 .../interactive_brokers/historic/client.py    | 58 -------------------
 nautilus_trader/persistence/wranglers_v2.py   | 43 +++++---------
 3 files changed, 14 insertions(+), 95 deletions(-)

diff --git a/examples/notebooks/backtest_example.ipynb b/examples/notebooks/backtest_example.ipynb
index f27db49c8893..e13608ab4924 100644
--- a/examples/notebooks/backtest_example.ipynb
+++ b/examples/notebooks/backtest_example.ipynb
@@ -168,14 +168,6 @@
    "source": [
     "result"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "af22401c-4d5b-4a58-bb18-97f460cb284c",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
diff --git a/nautilus_trader/adapters/interactive_brokers/historic/client.py b/nautilus_trader/adapters/interactive_brokers/historic/client.py
index e4faeb6e3b85..52e7df1b963b 100644
--- a/nautilus_trader/adapters/interactive_brokers/historic/client.py
+++ b/nautilus_trader/adapters/interactive_brokers/historic/client.py
@@ -31,7 +31,6 @@
 from nautilus_trader.model.identifiers import TraderId
 from nautilus_trader.model.instruments.base import Instrument
 from nautilus_trader.msgbus.bus import MessageBus
-from nautilus_trader.persistence.catalog import ParquetDataCatalog
 
 
 class HistoricInteractiveBrokersClient:
@@ -484,60 +483,3 @@ def _calculate_duration_segments(
             results.append((minus_days_date, f"{seconds} S"))
 
         return results
-
-
-# will remove this post testing and review
-async def main():
-    contract = IBContract(
-        secType="STK",
-        symbol="AAPL",
-        exchange="SMART",
-        primaryExchange="NASDAQ",
-    )
-    instrument_id = "TSLA.NASDAQ"
-
-    client = HistoricInteractiveBrokersClient(port=4002, client_id=5)
-    await client._connect()
-    await asyncio.sleep(2)
-
-    instruments = await client.request_instruments(
-        contracts=[contract],
-        instrument_ids=[instrument_id],
-    )
-
-    bars = await client.request_bars(
-        bar_specifications=["1-DAY-LAST", "8-HOUR-MID"],
-        start_date_time=datetime.datetime(2022, 10, 15, 3),
-        end_date_time=datetime.datetime(2023, 11, 1),
-        tz_name="America/New_York",
-        contracts=[contract],
-        instrument_ids=[instrument_id],
-    )
-
-    trade_ticks = await client.request_ticks(
-        "TRADES",
-        start_date_time=datetime.datetime(2023, 11, 6, 10, 0),
-        end_date_time=datetime.datetime(2023, 11, 6, 10, 1),
-        tz_name="America/New_York",
-        contracts=[contract],
-        instrument_ids=[instrument_id],
-    )
-
-    quote_ticks = await client.request_ticks(
-        "BID_ASK",
-        start_date_time=datetime.datetime(2023, 11, 6, 10, 0),
-        end_date_time=datetime.datetime(2023, 11, 6, 10, 1),
-        tz_name="America/New_York",
-        contracts=[contract],
-        instrument_ids=[instrument_id],
-    )
-
-    catalog = ParquetDataCatalog("./catalog")
-    catalog.write_data(instruments)
-    catalog.write_data(bars)
-    catalog.write_data(trade_ticks)
-    catalog.write_data(quote_ticks)
-
-
-if __name__ == "__main__":
-    asyncio.run(main())
diff --git a/nautilus_trader/persistence/wranglers_v2.py b/nautilus_trader/persistence/wranglers_v2.py
index 067938360d72..95d5781fda65 100644
--- a/nautilus_trader/persistence/wranglers_v2.py
+++ b/nautilus_trader/persistence/wranglers_v2.py
@@ -65,13 +65,6 @@ def decode(k, v):
             **{k.decode(): decode(k, v) for k, v in metadata.items() if k not in cls.IGNORE_KEYS},
         )
 
-    def scale_column(
-        self,
-        column: pd.Series,
-        dtype: pd.core.arrays.integer.IntegerDtype,
-    ) -> pd.Series:
-        return (column * 1e9).round().astype(dtype())
-
 
 class OrderBookDeltaDataWrangler(WranglerBase):
     """
@@ -146,10 +139,10 @@ def from_pandas(
         )
 
         # Scale prices and quantities
-        df["price"] = super().scale_column(df["price"], pd.Int64Dtype)
-        df["size"] = super().scale_column(df["size"], pd.UInt64Dtype)
+        df["price"] = (df["price"] * 1e9).astype(pd.Int64Dtype())
+        df["size"] = (df["size"] * 1e9).round().astype(pd.UInt64Dtype())
 
-        df["order_id"] = super().scale_column(df["order_id"], pd.UInt64Dtype)
+        df["order_id"] = df["order_id"].astype(pd.UInt64Dtype())
 
         # Process timestamps
         df["ts_event"] = (
@@ -254,17 +247,17 @@ def from_pandas(
         )
 
         # Scale prices and quantities
-        df["bid_price"] = super().scale_column(df["bid_price"], pd.Int64Dtype)
-        df["ask_price"] = super().scale_column(df["ask_price"], pd.Int64Dtype)
+        df["bid_price"] = (df["bid_price"] * 1e9).astype(pd.Int64Dtype())
+        df["ask_price"] = (df["ask_price"] * 1e9).astype(pd.Int64Dtype())
 
         # Create bid_size and ask_size columns
         if "bid_size" in df.columns:
-            df["bid_size"] = super().scale_column(df["bid_size"], pd.Int64Dtype)
+            df["bid_size"] = (df["bid_size"] * 1e9).astype(pd.Int64Dtype())
         else:
             df["bid_size"] = pd.Series([default_size * 1e9] * len(df), dtype=pd.UInt64Dtype())
 
         if "ask_size" in df.columns:
-            df["ask_size"] = super().scale_column(df["ask_size"], pd.Int64Dtype)
+            df["ask_size"] = (df["ask_size"] * 1e9).astype(pd.Int64Dtype())
         else:
             df["ask_size"] = pd.Series([default_size * 1e9] * len(df), dtype=pd.UInt64Dtype())
 
@@ -375,8 +368,8 @@ def from_pandas(
         )
 
         # Scale prices and quantities
-        df["price"] = super().scale_column(df["price"], pd.Int64Dtype)
-        df["size"] = super().scale_column(df["size"], pd.UInt64Dtype)
+        df["price"] = (df["price"] * 1e9).astype(pd.Int64Dtype())
+        df["size"] = (df["size"] * 1e9).round().astype(pd.UInt64Dtype())
 
         df["aggressor_side"] = df["aggressor_side"].map(_map_aggressor_side).astype(pd.UInt8Dtype())
         df["trade_id"] = df["trade_id"].astype(str)
@@ -488,23 +481,15 @@ def from_pandas(
         # Rename column
         df = df.rename(columns={"timestamp": "ts_event"})
 
-        # Check required columns
-        required_columns = {"open", "high", "low", "close", "ts_event"}
-        missing_columns = required_columns - set(df.columns)
-        if missing_columns:
-            raise ValueError(f"Missing columns: {missing_columns}")
-
-        # Scale OHLC
-        df["open"] = super().scale_column(df["open"], pd.Int64Dtype)
-        df["high"] = super().scale_column(df["high"], pd.Int64Dtype)
-        df["low"] = super().scale_column(df["low"], pd.Int64Dtype)
-        df["close"] = super().scale_column(df["close"], pd.Int64Dtype)
+        # Scale prices and quantities
+        df["open"] = (df["open"] * 1e9).astype(pd.Int64Dtype())
+        df["high"] = (df["high"] * 1e9).astype(pd.Int64Dtype())
+        df["low"] = (df["low"] * 1e9).astype(pd.Int64Dtype())
+        df["clow"] = (df["close"] * 1e9).astype(pd.Int64Dtype())
 
         if "volume" not in df.columns:
             df["volume"] = pd.Series([default_volume * 1e9] * len(df), dtype=pd.UInt64Dtype())
 
-        df["volume"] = super().scale_column(df["volume"], pd.UInt64Dtype)
-
         # Process timestamps
         df["ts_event"] = (
             pd.to_datetime(df["ts_event"], utc=True, format="mixed")

From 86fe191ebb1f1e972cef19ece8cd679b7d26f6f4 Mon Sep 17 00:00:00 2001
From: Ben Singleton <benfsingleton@gmail.com>
Date: Fri, 10 Nov 2023 23:22:48 -0500
Subject: [PATCH 5/6] tweaks

---
 docs/integrations/ib.md | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/docs/integrations/ib.md b/docs/integrations/ib.md
index 149a4a52d54b..5636032227c0 100644
--- a/docs/integrations/ib.md
+++ b/docs/integrations/ib.md
@@ -67,12 +67,12 @@ The adapter includes several major components:
 
 ## Instruments & Contracts
 
-In IB, a NautilusTrader `Instrument` is equivalent to a Contract](https://interactivebrokers.github.io/tws-api/contracts.html). Contracts can be either a [basic contract](https://interactivebrokers.github.io/tws-api/classIBApi_1_1Contract.html) or a more [detailed](https://interactivebrokers.github.io/tws-api/classIBApi_1_1ContractDetails.html) version (ContractDetails). The adapter models these using `IBContract` and `IBContractDetails` classes. The latter includes critical data like order types and trading hours, which are absent in the basic contract, making `IBContractDetails` convertible to an `Instrument`, unlike IBContract.
+In IB, a NautilusTrader `Instrument` is equivalent to a [Contract](https://interactivebrokers.github.io/tws-api/contracts.html). Contracts can be either a [basic contract](https://interactivebrokers.github.io/tws-api/classIBApi_1_1Contract.html) or a more [detailed](https://interactivebrokers.github.io/tws-api/classIBApi_1_1ContractDetails.html) version (ContractDetails). The adapter models these using `IBContract` and `IBContractDetails` classes. The latter includes critical data like order types and trading hours, which are absent in the basic contract. As a result, `IBContractDetails` can be converted to an `Instrument` while `IBContract` cannot.
 
 To search for contract information, use the [IB Contract Information Center](https://pennies.interactivebrokers.com/cstools/contract_info/).
 
 Examples of `IBContracts`:
-```
+```python
 from nautilus_trader.adapters.interactive_brokers.common import IBContract
 
 # Stock
@@ -105,7 +105,7 @@ Here's an example of retrieving and saving instrument and bar data. A more compr
 
 ```python
 import datetime
-
+from nautilus_trader.adapters.interactive_brokers.common import IBContract
 from nautilus_trader.adapters.interactive_brokers.historic import HistoricInteractiveBrokersClient
 from nautilus_trader.persistence.catalog import ParquetDataCatalog
 
@@ -142,9 +142,9 @@ Engaging in live or paper trading requires constructing and running a `TradingNo
 
 ### InstrumentProvider
 
-The `InteractiveBrokersInstrumentProvider` class functions as a bridge for accessing financial instrument data from IB. Configurable through `InteractiveBrokersInstrumentProviderConfig`, it allows for the customization of various instrument type parameters. Additionally, this provider offers specialized methods to build and retrieve the entire futures and options chains for given underlying securities.
+The `InteractiveBrokersInstrumentProvider` class functions as a bridge for accessing financial instrument data from IB. Configurable through `InteractiveBrokersInstrumentProviderConfig`, it allows for the customization of various instrument type parameters. Additionally, this provider offers specialized methods to build and retrieve the entire futures and options chains.
 
-```
+```python
 from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersInstrumentProviderConfig
 
 
@@ -175,7 +175,7 @@ instrument_provider_config = InteractiveBrokersInstrumentProviderConfig(
 
 ### Data Client
 
-`InteractiveBrokersDataClient` interfaces with Interactive Brokers for streaming and retrieving market data. Upon connection, it configures the [market data type](https://interactivebrokers.github.io/tws-api/market_data_type.html) and loads instruments based on the settings in `InteractiveBrokersInstrumentProviderConfig`. This client can subscribe to and unsubscribe from various market data types, including quote ticks, trade ticks, and bars.
+`InteractiveBrokersDataClient` interfaces with IB for streaming and retrieving market data. Upon connection, it configures the [market data type](https://interactivebrokers.github.io/tws-api/market_data_type.html) and loads instruments based on the settings in `InteractiveBrokersInstrumentProviderConfig`. This client can subscribe to and unsubscribe from various market data types, including quote ticks, trade ticks, and bars.
 
 Configurable through `InteractiveBrokersDataClientConfig`, it allows adjustments for handling revised bars, trading hours preferences, and market data types (e.g., `IBMarketDataTypeEnum.REALTIME` or `IBMarketDataTypeEnum.DELAYED_FROZEN`).
 
@@ -218,7 +218,7 @@ exec_client_config = InteractiveBrokersExecClientConfig(
 
 Setting up a complete trading environment typically involves configuring a `TradingNodeConfig`, which includes data and execution client configurations. Additional configurations are specified in `LiveDataEngineConfig` to accommodate IB-specific requirements. A `TradingNode` is then instantiated from these configurations, and factories for creating `InteractiveBrokersDataClient` and `InteractiveBrokersExecutionClient` are added. Finally, the node is built and run.
 
-For a comprehensive example, refer to this [guide](https://github.com/nautechsystems/nautilus_trader/blob/master/examples/live/interactive_brokers/interactive_brokers_example.py).
+For a comprehensive example, refer to this [script](https://github.com/nautechsystems/nautilus_trader/blob/master/examples/live/interactive_brokers/interactive_brokers_example.py).
 
 
 ```python

From 95a9763b88538369e77946d053e1629f6be6771d Mon Sep 17 00:00:00 2001
From: Ben Singleton <benfsingleton@gmail.com>
Date: Sat, 11 Nov 2023 13:20:57 -0500
Subject: [PATCH 6/6] fix gateway

---
 docs/integrations/ib.md                           |  2 +-
 .../live/interactive_brokers/historic_download.py | 15 ++++++++++++++-
 .../adapters/interactive_brokers/client/client.py |  3 ++-
 .../adapters/interactive_brokers/config.py        |  6 +++++-
 .../adapters/interactive_brokers/gateway.py       | 14 +++++++++-----
 .../interactive_brokers/historic/client.py        |  6 +++++-
 6 files changed, 36 insertions(+), 10 deletions(-)

diff --git a/docs/integrations/ib.md b/docs/integrations/ib.md
index 5636032227c0..a4a8a09bb684 100644
--- a/docs/integrations/ib.md
+++ b/docs/integrations/ib.md
@@ -4,7 +4,7 @@ Interactive Brokers (IB) is a trading platform that allows trading in a wide ran
 
 The TWS API serves as an interface to IB's standalone trading applications: TWS and IB Gateway. Both can be downloaded from the IB website. If you haven't installed TWS or IB Gateway yet, refer to the [Initial Setup](https://interactivebrokers.github.io/tws-api/initial_setup.html) guide. In NautilusTrader, you'll establish a connection to one of these applications via the `InteractiveBrokersClient`.
 
-Alternatively, you can start with a [dockerized version](https://github.com/unusualalpha/ib-gateway-docker/pkgs/container/ib-gateway) of the IB Gateway, particularly useful when deploying trading strategies on a hosted cloud platform. This requires having [Docker](https://www.docker.com/) installed on your machine, along with the [docker](https://pypi.org/project/docker/) Python package, which NautilusTrader conveniently includes as an extra package.
+Alternatively, you can start with a [dockerized version](https://github.com/gnzsnz/ib-gateway-docker) of the IB Gateway, particularly useful when deploying trading strategies on a hosted cloud platform. This requires having [Docker](https://www.docker.com/) installed on your machine, along with the [docker](https://pypi.org/project/docker/) Python package, which NautilusTrader conveniently includes as an extra package.
 
 **Note**: The standalone TWS and IB Gateway applications necessitate manual input of username, password, and trading mode (live or paper) at startup. The dockerized version of the IB Gateway handles these steps programmatically.
 
diff --git a/examples/live/interactive_brokers/historic_download.py b/examples/live/interactive_brokers/historic_download.py
index afcd2980ad0f..e7af08ba7829 100644
--- a/examples/live/interactive_brokers/historic_download.py
+++ b/examples/live/interactive_brokers/historic_download.py
@@ -15,13 +15,24 @@
 # -------------------------------------------------------------------------------------------------
 import asyncio
 import datetime
+import os
 
 from nautilus_trader.adapters.interactive_brokers.common import IBContract
+from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersGatewayConfig
+from nautilus_trader.adapters.interactive_brokers.gateway import InteractiveBrokersGateway
 from nautilus_trader.adapters.interactive_brokers.historic import HistoricInteractiveBrokersClient
 from nautilus_trader.persistence.catalog import ParquetDataCatalog
 
 
 async def main():
+    gateway_config = InteractiveBrokersGatewayConfig(
+        username=os.environ["TWS_USERNAME"],
+        password=os.environ["TWS_PASSWORD"],
+        port=4002,
+    )
+    gateway = InteractiveBrokersGateway(config=gateway_config)
+    gateway.start()
+
     contract = IBContract(
         secType="STK",
         symbol="AAPL",
@@ -41,9 +52,9 @@ async def main():
 
     bars = await client.request_bars(
         bar_specifications=["1-HOUR-LAST", "30-MINUTE-MID"],
+        start_date_time=datetime.datetime(2023, 11, 6, 9, 30),
         end_date_time=datetime.datetime(2023, 11, 6, 16, 30),
         tz_name="America/New_York",
-        duration="1 D",
         contracts=[contract],
         instrument_ids=[instrument_id],
     )
@@ -66,6 +77,8 @@ async def main():
         instrument_ids=[instrument_id],
     )
 
+    gateway.stop()
+
     catalog = ParquetDataCatalog("./catalog")
     catalog.write_data(instruments)
     catalog.write_data(bars)
diff --git a/nautilus_trader/adapters/interactive_brokers/client/client.py b/nautilus_trader/adapters/interactive_brokers/client/client.py
index 20952aa43d05..922368248573 100644
--- a/nautilus_trader/adapters/interactive_brokers/client/client.py
+++ b/nautilus_trader/adapters/interactive_brokers/client/client.py
@@ -1298,6 +1298,7 @@ async def get_historical_ticks(
         start_date_time: pd.Timestamp | str = "",
         end_date_time: pd.Timestamp | str = "",
         use_rth: bool = True,
+        timeout: int = 60,
     ):
         if isinstance(start_date_time, pd.Timestamp):
             start_date_time = start_date_time.strftime("%Y%m%d %H:%M:%S %Z")
@@ -1325,7 +1326,7 @@ async def get_historical_ticks(
                 cancel=functools.partial(self._client.cancelHistoricalData, reqId=req_id),
             )
             request.handle()
-            return await self._await_request(request, 60)
+            return await self._await_request(request, timeout)
         else:
             self._log.info(f"Request already exist for {request}")
 
diff --git a/nautilus_trader/adapters/interactive_brokers/config.py b/nautilus_trader/adapters/interactive_brokers/config.py
index 8bd4692c62bf..514e79f7be09 100644
--- a/nautilus_trader/adapters/interactive_brokers/config.py
+++ b/nautilus_trader/adapters/interactive_brokers/config.py
@@ -36,6 +36,10 @@ class InteractiveBrokersGatewayConfig(NautilusConfig, frozen=True):
     password : str, optional
         The Interactive Brokers account password.
         If ``None`` then will source the `TWS_PASSWORD`.
+    host : str, optional
+        The hostname or ip address for the IB Gateway or TWS.
+    port : int, optional
+        The port for the gateway server ("paper" 4002, or "live" 4001).
     trading_mode: str
         paper or live.
     start: bool, optional
@@ -49,7 +53,7 @@ class InteractiveBrokersGatewayConfig(NautilusConfig, frozen=True):
 
     username: str | None = None
     password: str | None = None
-    host: str | None = "localhost"
+    host: str | None = "127.0.0.1"
     port: Literal[4001, 4002] | None = None
     trading_mode: Literal["paper", "live"] = "paper"
     start: bool = False
diff --git a/nautilus_trader/adapters/interactive_brokers/gateway.py b/nautilus_trader/adapters/interactive_brokers/gateway.py
index c808323d45da..58519f1841ff 100644
--- a/nautilus_trader/adapters/interactive_brokers/gateway.py
+++ b/nautilus_trader/adapters/interactive_brokers/gateway.py
@@ -43,7 +43,7 @@ class InteractiveBrokersGateway:
     A class to manage starting an Interactive Brokers Gateway docker container.
     """
 
-    IMAGE: ClassVar[str] = "ghcr.io/unusualalpha/ib-gateway:10.19"
+    IMAGE: ClassVar[str] = "ghcr.io/gnzsnz/ib-gateway:stable"
     CONTAINER_NAME: ClassVar[str] = "nautilus-ib-gateway"
     PORTS: ClassVar[dict[str, int]] = {"paper": 4002, "live": 4001}
 
@@ -51,7 +51,7 @@ def __init__(
         self,
         username: str | None = None,
         password: str | None = None,
-        host: str | None = "localhost",
+        host: str | None = "127.0.0.1",
         port: int | None = None,
         trading_mode: Literal["paper", "live"] | None = "paper",
         start: bool = False,
@@ -72,9 +72,9 @@ def __init__(
 
         self.username = username or os.getenv("TWS_USERNAME")
         self.password = password or os.getenv("TWS_PASSWORD")
-        if username is None:
+        if self.username is None:
             raise ValueError("`username` not set nor available in env `TWS_USERNAME`")
-        if password is None:
+        if self.password is None:
             raise ValueError("`password` not set nor available in env `TWS_PASSWORD`")
 
         self.trading_mode = trading_mode
@@ -158,7 +158,11 @@ def start(self, wait: int | None = 90):
             name=f"{self.CONTAINER_NAME}-{self.port}",
             restart_policy={"Name": "always"},
             detach=True,
-            ports={str(self.port): self.PORTS[self.trading_mode], str(self.port + 100): "5900"},
+            ports={
+                "4003": (self.host, 4001),
+                "4004": (self.host, 4002),
+                "5900": (self.host, 5900),
+            },
             platform="amd64",
             environment={
                 "TWS_USERID": self.username,
diff --git a/nautilus_trader/adapters/interactive_brokers/historic/client.py b/nautilus_trader/adapters/interactive_brokers/historic/client.py
index 52e7df1b963b..ba1b960c0aba 100644
--- a/nautilus_trader/adapters/interactive_brokers/historic/client.py
+++ b/nautilus_trader/adapters/interactive_brokers/historic/client.py
@@ -212,7 +212,7 @@ async def request_bars(
                 ):
                     self.log.info(
                         f"{instrument_id}: Requesting historical bars: {bar_type} ending on '{segment_end_date_time}' "
-                        "with duration '{segment_duration}'",
+                        f"with duration '{segment_duration}'",
                     )
 
                     bars = await self._client.get_historical_bars(
@@ -243,6 +243,7 @@ async def request_ticks(
         contracts: list[IBContract] | None = None,
         instrument_ids: list[str] | None = None,
         use_rth: bool = True,
+        timeout: int = 60,
     ) -> list[TradeTick | QuoteTick]:
         """
         Return TradeTicks or QuoteTicks for one or more bar specifications for a list of
@@ -264,6 +265,8 @@ async def request_ticks(
             Instrument IDs (e.g. AAPL.NASDAQ) defining which ticks to retrieve.
         use_rth : bool, default 'True'
             Whether to use regular trading hours.
+        timeout : int, default '60'
+            The timeout in seconds for each request.
 
         Returns
         -------
@@ -316,6 +319,7 @@ async def request_ticks(
                     tick_type=tick_type,
                     start_date_time=current_start_date_time,
                     use_rth=use_rth,
+                    timeout=timeout,
                 )
 
                 if not ticks: