For code/output blocks: Use ``` (aka backtick or grave accent) in a single line before and after the block. See: http://commonmark.org/help/

Getting IB data source to work



  • Hi,

    I have built all my indicators, analyzers, and observers, run my backtests, and now I'm ready to look at deploying a strategy live. I've been trying for the past few days now to integrate Interactive Brokers with Backtrader to just start receiving some historical data - I've tried following all the sample scripts, blog posts, medium posts, docs, everything. However, I just cannot get it to return data other than anything FX (e.g. EUR.USD-CASH-IDEALPRO). See below for the ibtest.py version I'm using, although as mentioned above, I have tried all of the different code samples.

    from __future__ import (absolute_import, division, print_function,
                            unicode_literals)
    
    import argparse
    import datetime
    
    # The above could be sent to an independent module
    import backtrader as bt
    from backtrader.utils import flushfile  # win32 quick stdout flushing
    
    
    class TestStrategy(bt.Strategy):
        params = dict(
            smaperiod=5,
            trade=False,
            stake=10,
            exectype=bt.Order.Market,
            stopafter=0,
            valid=None,
            cancel=0,
            donotsell=False,
        )
    
        def __init__(self):
            # To control operation entries
            self.orderid = list()
            self.order = None
    
            self.counttostop = 0
            self.datastatus = 0
    
            # Create SMA on 2nd data
            self.sma = bt.indicators.MovAv.SMA(self.data, period=self.p.smaperiod)
    
            print('--------------------------------------------------')
            print('Strategy Created')
            print('--------------------------------------------------')
    
        def notify_data(self, data, status, *args, **kwargs):
            print('*' * 5, 'DATA NOTIF:', data._getstatusname(status), *args)
            if status == data.LIVE:
                self.counttostop = self.p.stopafter
                self.datastatus = 1
    
        def notify_store(self, msg, *args, **kwargs):
            print('*' * 5, 'STORE NOTIF:', msg)
    
        def notify_order(self, order):
            if order.status in [order.Completed, order.Cancelled, order.Rejected]:
                self.order = None
    
            print('-' * 50, 'ORDER BEGIN', datetime.datetime.now())
            print(order)
            print('-' * 50, 'ORDER END')
    
        def notify_trade(self, trade):
            print('-' * 50, 'TRADE BEGIN', datetime.datetime.now())
            print(trade)
            print('-' * 50, 'TRADE END')
    
        def prenext(self):
            self.next(frompre=True)
    
        def next(self, frompre=False):
            txt = list()
            txt.append('%04d' % len(self))
            dtfmt = '%Y-%m-%dT%H:%M:%S.%f'
            txt.append('%s' % self.data.datetime.datetime(0).strftime(dtfmt))
            txt.append('O: {:.5f}'.format(self.data.open[0]))
            txt.append('H: {:.5f}'.format(self.data.high[0]))
            txt.append('L: {:.5f}'.format(self.data.low[0]))
            txt.append('C: {:.5f}'.format(self.data.close[0]))
            # txt.append('{}'.format(self.data.volume[0]))
            # txt.append('{}'.format(self.data.openinterest[0]))
            # txt.append('{}'.format(self.sma[0]))
            print(', '.join(txt))
    
            if len(self.datas) > 1:
                txt = list()
                txt.append('%04d' % len(self))
                dtfmt = '%Y-%m-%dT%H:%M:%S.%f'
                txt.append('%s' % self.data1.datetime.datetime(0).strftime(dtfmt))
                txt.append('{}'.format(self.data1.open[0]))
                txt.append('{}'.format(self.data1.high[0]))
                txt.append('{}'.format(self.data1.low[0]))
                txt.append('{}'.format(self.data1.close[0]))
                txt.append('{}'.format(self.data1.volume[0]))
                txt.append('{}'.format(self.data1.openinterest[0]))
                txt.append('{}'.format(float('NaN')))
                print(', '.join(txt))
    
            if self.counttostop:  # stop after x live lines
                self.counttostop -= 1
                if not self.counttostop:
                    self.env.runstop()
                    return
    
            if not self.p.trade:
                return
    
            if self.datastatus and not self.position and len(self.orderid) < 1:
                self.order = self.buy(size=self.p.stake,
                                      exectype=self.p.exectype,
                                      price=round(self.data0.close[0] * 0.90, 2),
                                      valid=self.p.valid)
    
                self.orderid.append(self.order)
            elif self.position.size > 0 and not self.p.donotsell:
                if self.order is None:
                    self.order = self.sell(size=self.p.stake // 2,
                                           exectype=bt.Order.Market,
                                           price=self.data0.close[0])
    
            elif self.order is not None and self.p.cancel:
                if self.datastatus > self.p.cancel:
                    self.cancel(self.order)
    
            if self.datastatus:
                self.datastatus += 1
    
        def start(self):
            if self.data0.contractdetails is not None:
                print('Timezone from ContractDetails: {}'.format(
                      self.data0.contractdetails.m_timeZoneId))
    
            header = ['Datetime', 'Open', 'High', 'Low', 'Close', 'Volume',
                      'OpenInterest', 'SMA']
            print(', '.join(header))
    
            self.done = False
    
    
    def runstrategy():
        args = parse_args([
            '--port', '7497',
            '--data0', 'TWTR',
            # '--data0', 'EUR.USD-CASH-IDEALPRO',
            '--historical',
            '--fromdate', '2018-01-02',
            # '--no-backfill_start',
            # '--no-backfill',
            '--timeframe', bt.TimeFrame.Names[bt.TimeFrame.Days],
            '--compression', '1',
            '--timezone', 'NZ',
        ])
    
        # Create a cerebro
        cerebro = bt.Cerebro()
    
        storekwargs = dict(
            host=args.host, port=args.port,
            clientId=args.clientId, timeoffset=not args.no_timeoffset,
            reconnect=args.reconnect, timeout=args.timeout,
            notifyall=args.notifyall, _debug=args.debug
        )
    
        if args.usestore:
            ibstore = bt.stores.IBStore(**storekwargs)
    
        if args.broker:
            if args.usestore:
                broker = ibstore.getbroker()
            else:
                broker = bt.brokers.IBBroker(**storekwargs)
    
            cerebro.setbroker(broker)
    
        timeframe = bt.TimeFrame.TFrame(args.timeframe)
        if args.resample or args.replay:
            datatf = bt.TimeFrame.Ticks
            datacomp = 1
        else:
            datatf = timeframe
            datacomp = args.compression
    
        fromdate = None
        if args.fromdate:
            dtformat = '%Y-%m-%d' + ('T%H:%M:%S' * ('T' in args.fromdate))
            fromdate = datetime.datetime.strptime(args.fromdate, dtformat)
    
        IBDataFactory = ibstore.getdata if args.usestore else bt.feeds.IBData
    
        datakwargs = dict(
            timeframe=datatf, compression=datacomp,
            historical=args.historical, fromdate=fromdate,
            rtbar=args.rtbar,
            qcheck=args.qcheck,
            what=args.what,
            backfill_start=not args.no_backfill_start,
            backfill=not args.no_backfill,
            latethrough=args.latethrough,
            tz=args.timezone
        )
    
        if not args.usestore and not args.broker:   # neither store nor broker
            datakwargs.update(storekwargs)  # pass the store args over the data
    
        data0 = IBDataFactory(dataname=args.data0, **datakwargs)
    
        data1 = None
        if args.data1 is not None:
            data1 = IBDataFactory(dataname=args.data1, **datakwargs)
    
        rekwargs = dict(
            timeframe=timeframe, compression=args.compression,
            bar2edge=not args.no_bar2edge,
            adjbartime=not args.no_adjbartime,
            rightedge=not args.no_rightedge,
            takelate=not args.no_takelate,
        )
    
        if args.replay:
            cerebro.replaydata(dataname=data0, **rekwargs)
    
            if data1 is not None:
                cerebro.replaydata(dataname=data1, **rekwargs)
    
        elif args.resample:
            cerebro.resampledata(dataname=data0, **rekwargs)
    
            if data1 is not None:
                cerebro.resampledata(dataname=data1, **rekwargs)
    
        else:
            cerebro.adddata(data0)
            if data1 is not None:
                cerebro.adddata(data1)
    
        if args.valid is None:
            valid = None
        else:
            datetime.timedelta(seconds=args.valid)
        # Add the strategy
        cerebro.addstrategy(TestStrategy,
                            smaperiod=args.smaperiod,
                            trade=args.trade,
                            exectype=bt.Order.ExecType(args.exectype),
                            stake=args.stake,
                            stopafter=args.stopafter,
                            valid=valid,
                            cancel=args.cancel,
                            donotsell=args.donotsell)
    
        # Live data ... avoid long data accumulation by switching to "exactbars"
        cerebro.run(exactbars=args.exactbars)
    
        if args.plot and args.exactbars < 1:  # plot if possible
            cerebro.plot()
    
    
    def parse_args(pargs=None):
        parser = argparse.ArgumentParser(
            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
            description='Test Interactive Brokers integration')
    
        parser.add_argument('--exactbars', default=1, type=int,
                            required=False, action='store',
                            help='exactbars level, use 0/-1/-2 to enable plotting')
    
        parser.add_argument('--plot',
                            required=False, action='store_true',
                            help='Plot if possible')
    
        parser.add_argument('--stopafter', default=0, type=int,
                            required=False, action='store',
                            help='Stop after x lines of LIVE data')
    
        parser.add_argument('--usestore',
                            required=False, action='store_true',
                            help='Use the store pattern')
    
        parser.add_argument('--notifyall',
                            required=False, action='store_true',
                            help='Notify all messages to strategy as store notifs')
    
        parser.add_argument('--debug',
                            required=False, action='store_true',
                            help='Display all info received form IB')
    
        parser.add_argument('--host', default='127.0.0.1',
                            required=False, action='store',
                            help='Host for the Interactive Brokers TWS Connection')
    
        parser.add_argument('--qcheck', default=0.5, type=float,
                            required=False, action='store',
                            help=('Timeout for periodic '
                                  'notification/resampling/replaying check'))
    
        parser.add_argument('--port', default=7496, type=int,
                            required=False, action='store',
                            help='Port for the Interactive Brokers TWS Connection')
    
        parser.add_argument('--clientId', default=None, type=int,
                            required=False, action='store',
                            help='Client Id to connect to TWS (default: random)')
    
        parser.add_argument('--no-timeoffset',
                            required=False, action='store_true',
                            help=('Do not Use TWS/System time offset for non '
                                  'timestamped prices and to align resampling'))
    
        parser.add_argument('--reconnect', default=3, type=int,
                            required=False, action='store',
                            help='Number of recconnection attempts to TWS')
    
        parser.add_argument('--timeout', default=3.0, type=float,
                            required=False, action='store',
                            help='Timeout between reconnection attempts to TWS')
    
        parser.add_argument('--data0', default=None,
                            required=True, action='store',
                            help='data 0 into the system')
    
        parser.add_argument('--data1', default=None,
                            required=False, action='store',
                            help='data 1 into the system')
    
        parser.add_argument('--timezone', default=None,
                            required=False, action='store',
                            help='timezone to get time output into (pytz names)')
    
        parser.add_argument('--what', default=None,
                            required=False, action='store',
                            help='specific price type for historical requests')
    
        parser.add_argument('--no-backfill_start',
                            required=False, action='store_true',
                            help='Disable backfilling at the start')
    
        parser.add_argument('--latethrough',
                            required=False, action='store_true',
                            help=('if resampling replaying, adjusting time '
                                  'and disabling time offset, let late samples '
                                  'through'))
    
        parser.add_argument('--no-backfill',
                            required=False, action='store_true',
                            help='Disable backfilling after a disconnection')
    
        parser.add_argument('--rtbar', default=False,
                            required=False, action='store_true',
                            help='Use 5 seconds real time bar updates if possible')
    
        parser.add_argument('--historical',
                            required=False, action='store_true',
                            help='do only historical download')
    
        parser.add_argument('--fromdate',
                            required=False, action='store',
                            help=('Starting date for historical download '
                                  'with format: YYYY-MM-DD[THH:MM:SS]'))
    
        parser.add_argument('--smaperiod', default=5, type=int,
                            required=False, action='store',
                            help='Period to apply to the Simple Moving Average')
    
        pgroup = parser.add_mutually_exclusive_group(required=False)
    
        pgroup.add_argument('--replay',
                            required=False, action='store_true',
                            help='replay to chosen timeframe')
    
        pgroup.add_argument('--resample',
                            required=False, action='store_true',
                            help='resample to chosen timeframe')
    
        parser.add_argument('--timeframe', default=bt.TimeFrame.Names[0],
                            choices=bt.TimeFrame.Names,
                            required=False, action='store',
                            help='TimeFrame for Resample/Replay')
    
        parser.add_argument('--compression', default=1, type=int,
                            required=False, action='store',
                            help='Compression for Resample/Replay')
    
        parser.add_argument('--no-takelate',
                            required=False, action='store_true',
                            help=('resample/replay, do not accept late samples '
                                  'in new bar if the data source let them through '
                                  '(latethrough)'))
    
        parser.add_argument('--no-bar2edge',
                            required=False, action='store_true',
                            help='no bar2edge for resample/replay')
    
        parser.add_argument('--no-adjbartime',
                            required=False, action='store_true',
                            help='no adjbartime for resample/replay')
    
        parser.add_argument('--no-rightedge',
                            required=False, action='store_true',
                            help='no rightedge for resample/replay')
    
        parser.add_argument('--broker',
                            required=False, action='store_true',
                            help='Use IB as broker')
    
        parser.add_argument('--trade',
                            required=False, action='store_true',
                            help='Do Sample Buy/Sell operations')
    
        parser.add_argument('--donotsell',
                            required=False, action='store_true',
                            help='Do not sell after a buy')
    
        parser.add_argument('--exectype', default=bt.Order.ExecTypes[0],
                            choices=bt.Order.ExecTypes,
                            required=False, action='store',
                            help='Execution to Use when opening position')
    
        parser.add_argument('--stake', default=10, type=int,
                            required=False, action='store',
                            help='Stake to use in buy operations')
    
        parser.add_argument('--valid', default=None, type=int,
                            required=False, action='store',
                            help='Seconds to keep the order alive (0 means DAY)')
    
        parser.add_argument('--cancel', default=0, type=int,
                            required=False, action='store',
                            help=('Cancel a buy order after n bars in operation,'
                                  ' to be combined with orders like Limit'))
    
        if pargs is not None:
            return parser.parse_args(pargs)
    
        return parser.parse_args()
    
    
    if __name__ == '__main__':
        runstrategy()
    

    Running this script, and the others that have been "shown" to work, results in:

    Server Version: 76
    TWS Time at connection:20200205 16:02:32 NZST
    --------------------------------------------------
    Strategy Created
    --------------------------------------------------
    Timezone from ContractDetails: EST (Eastern Standard Time)
    Datetime, Open, High, Low, Close, Volume, OpenInterest, SMA
    ***** STORE NOTIF: <error id=-1, errorCode=2104, errorMsg=Market data farm connection is OK:cashfarm>
    ***** STORE NOTIF: <error id=-1, errorCode=2104, errorMsg=Market data farm connection is OK:usfarm>
    ***** STORE NOTIF: <error id=-1, errorCode=2106, errorMsg=HMDS data farm connection is OK:cashhmds>
    ***** STORE NOTIF: <error id=-1, errorCode=2106, errorMsg=HMDS data farm connection is OK:hkhmds>
    ***** STORE NOTIF: <error id=-1, errorCode=2106, errorMsg=HMDS data farm connection is OK:ushmds>
    ***** STORE NOTIF: <error id=-1, errorCode=2158, errorMsg=Sec-def data farm connection is OK:secdefhk>
    ***** DATA NOTIF: DELAYED
    ***** DATA NOTIF: DISCONNECTED
    ***** STORE NOTIF: <error id=16777217, errorCode=162, errorMsg=Historical Market Data Service error message:No market data permissions for NYSE STK>
    
    Process finished with exit code 0
    

    I just cannot for the life of me figure out why I cannot get any historical data - inside TWS (and the STORE NOTIFs above), all the data connections are working. I've tried following all of the sample scripts (even uninstalling and reinstalling TWS to "perfectly" follow along with the examples), and while those are able to get the data coming through, mine never seem too.

    Does anybody have any piece of information that may help me solve this issue? I really don't know what to do from here...

    Many thanks in advance!



  • Hi, IB tells you "No market data permissions for NYSE STK". Have you subscribed to NYSE stocks? If not, that is probably the root cause. If I remember correctly, seeing hist data on TWS and getting this data through API are 2 different things from a subscription perspective.

    Regards,
    Lamp'


Log in to reply
 

});