[Code snippet] Copying IB data to Pandas for efficiency

  • There's a small inefficiency issue when optimizing parameters using Interactive Brokers historical data, whether it's with backtrader's optimization module or an external optimization library such as Optunity. The issue is that a new connection is made to download the data from IB for each iteration of each permutation in parameter ranges.

    To speed things up, I use the IbPy library to copy the data into a pandas dataframe then pass this over to backtrader as a data feed. In this way, the IB data is read once from TWS, but is made available to the optimizer via a data frame in memory as often as the optimization engine requires it.

    There may be a simpler way of doing this with ibStore, but if there is I wan't able to figure it out.

    Code snippet for the IbPy/pandas solution is below using backtrader's optimization example code as a base...

    import os.path  # To manage paths
    import sys  # To find out the script name (in argv[0])
    import backtrader as bt
    from datetime import datetime
    import pytz, tzlocal
    from time import sleep, strftime, localtime  
    from ib.ext.Contract import Contract  
    from ib.opt import ibConnection, message
    import pandas as pd
    # Set up IB message handler to dump to pandas dataframe
    df = pd.DataFrame( columns=['Date', 'Open', 'High', 'Low', 'Close', 'Volume', 'OpenInterest'])
    s = pd.Series()
    # define historical data handler for IB - this will populate our pandas data frame
    def historical_data_handler(msg):  
        global df
    #    print (msg.reqId, msg.date, msg.open, msg.close, msg.high, msg.low)
        if ('finished' in str(msg.date)) == False:
            s = ([datetime.fromtimestamp(int(msg.date)), msg.open, msg.high, msg.low, msg.close, msg.volume, 0])
            df.loc[len(df)] = s
    con = ibConnection(host='',port=7496,clientId=77)
    con.register(historical_data_handler, message.historicalData)
    # IBpy - set up contract details and historical data request
    qqq = Contract()  
    qqq.m_symbol = 'ES'
    qqq.m_secType = 'FUT'  
    qqq.m_exchange = 'GLOBEX'  
    qqq.m_currency = 'USD'
    qqq.m_expiry = '201709'
    con.reqHistoricalData(0, qqq, '', '3 W', '1 hour', 'TRADES', 1, 2)
    data = bt.feeds.PandasData(dataname = df,  tz=pytz.timezone('US/Eastern'))
    # assign our newly created dataframe to a bt.feed
    data = bt.feeds.PandasData(dataname = df,  tz=pytz.timezone('US/Eastern'))
    # Create a Strategy
    class TestStrategy(bt.Strategy):
        params = (
            ('maperiod', 15),
            ('printlog', False),
        def log(self, txt, dt=None, doprint=False):
            ''' Logging function fot this strategy'''
            if self.params.printlog or doprint:
                dt = dt or self.datas[0].datetime.date(0)
                print('%s, %s' % (dt.isoformat(), txt))
        def __init__(self):
            # Keep a reference to the "close" line in the data[0] dataseries
            self.dataclose = self.datas[0].close
            # To keep track of pending orders and buy price/commission
            self.order = None
            self.buyprice = None
            self.buycomm = None
            # Add a MovingAverageSimple indicator
            self.sma = bt.indicators.SimpleMovingAverage(
                self.datas[0], period=self.params.maperiod)
        def notify_order(self, order):
            if order.status in [order.Submitted, order.Accepted]:
                # Buy/Sell order submitted/accepted to/by broker - Nothing to do
            # Check if an order has been completed
            # Attention: broker could reject order if not enougth cash
            if order.status in [order.Completed]:
                if order.isbuy():
                        'BUY EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' %
                    self.buyprice = order.executed.price
                    self.buycomm = order.executed.comm
                else:  # Sell
                    self.log('SELL EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' %
                self.bar_executed = len(self)
            elif order.status in [order.Canceled, order.Margin, order.Rejected]:
                self.log('Order Canceled/Margin/Rejected')
            # Write down: no pending order
            self.order = None
        def notify_trade(self, trade):
            if not trade.isclosed:
            self.log('OPERATION PROFIT, GROSS %.2f, NET %.2f' %
                     (trade.pnl, trade.pnlcomm))
        def next(self):
            # Simply log the closing price of the series from the reference
            self.log('Close, %.2f' % self.dataclose[0])
            # Check if an order is pending ... if yes, we cannot send a 2nd one
            if self.order:
            # Check if we are in the market
            if not self.position:
                # Not yet ... we MIGHT BUY if ...
                if self.dataclose[0] > self.sma[0]:
                    # BUY, BUY, BUY!!! (with all possible default parameters)
                    self.log('BUY CREATE, %.2f' % self.dataclose[0])
                    # Keep track of the created order to avoid a 2nd order
                    self.order = self.buy()
                if self.dataclose[0] < self.sma[0]:
                    # SELL, SELL, SELL!!! (with all possible default parameters)
                    self.log('SELL CREATE, %.2f' % self.dataclose[0])
                    # Keep track of the created order to avoid a 2nd order
                    self.order = self.sell()
        def stop(self):
            self.log('(MA Period %2d) Ending Value %.2f' %
                     (self.params.maperiod, self.broker.getvalue()), doprint=True)
    # Create a cerebro entity
    cerebro = bt.Cerebro(maxcpus=1)
    # Add the Data Feed to Cerebro
    # Add a strategy
    strats = cerebro.optstrategy(
        maperiod=range(5, 10))
    # Run over everything

Log in to reply

Looks like your connection to Backtrader Community was lost, please wait while we try to reconnect.