Hi,
I have built all my indicators, analyzers, and observers, run my backtests, and now I'm ready to look at deploying a strategy live. I've been trying for the past few days now to integrate Interactive Brokers with Backtrader to just start receiving some historical data - I've tried following all the sample scripts, blog posts, medium posts, docs, everything. However, I just cannot get it to return data other than anything FX (e.g. EUR.USD-CASH-IDEALPRO). See below for the ibtest.py version I'm using, although as mentioned above, I have tried all of the different code samples.
from __future__ import (absolute_import, division, print_function,
unicode_literals)
import argparse
import datetime
# The above could be sent to an independent module
import backtrader as bt
from backtrader.utils import flushfile # win32 quick stdout flushing
class TestStrategy(bt.Strategy):
params = dict(
smaperiod=5,
trade=False,
stake=10,
exectype=bt.Order.Market,
stopafter=0,
valid=None,
cancel=0,
donotsell=False,
)
def __init__(self):
# To control operation entries
self.orderid = list()
self.order = None
self.counttostop = 0
self.datastatus = 0
# Create SMA on 2nd data
self.sma = bt.indicators.MovAv.SMA(self.data, period=self.p.smaperiod)
print('--------------------------------------------------')
print('Strategy Created')
print('--------------------------------------------------')
def notify_data(self, data, status, *args, **kwargs):
print('*' * 5, 'DATA NOTIF:', data._getstatusname(status), *args)
if status == data.LIVE:
self.counttostop = self.p.stopafter
self.datastatus = 1
def notify_store(self, msg, *args, **kwargs):
print('*' * 5, 'STORE NOTIF:', msg)
def notify_order(self, order):
if order.status in [order.Completed, order.Cancelled, order.Rejected]:
self.order = None
print('-' * 50, 'ORDER BEGIN', datetime.datetime.now())
print(order)
print('-' * 50, 'ORDER END')
def notify_trade(self, trade):
print('-' * 50, 'TRADE BEGIN', datetime.datetime.now())
print(trade)
print('-' * 50, 'TRADE END')
def prenext(self):
self.next(frompre=True)
def next(self, frompre=False):
txt = list()
txt.append('%04d' % len(self))
dtfmt = '%Y-%m-%dT%H:%M:%S.%f'
txt.append('%s' % self.data.datetime.datetime(0).strftime(dtfmt))
txt.append('O: {:.5f}'.format(self.data.open[0]))
txt.append('H: {:.5f}'.format(self.data.high[0]))
txt.append('L: {:.5f}'.format(self.data.low[0]))
txt.append('C: {:.5f}'.format(self.data.close[0]))
# txt.append('{}'.format(self.data.volume[0]))
# txt.append('{}'.format(self.data.openinterest[0]))
# txt.append('{}'.format(self.sma[0]))
print(', '.join(txt))
if len(self.datas) > 1:
txt = list()
txt.append('%04d' % len(self))
dtfmt = '%Y-%m-%dT%H:%M:%S.%f'
txt.append('%s' % self.data1.datetime.datetime(0).strftime(dtfmt))
txt.append('{}'.format(self.data1.open[0]))
txt.append('{}'.format(self.data1.high[0]))
txt.append('{}'.format(self.data1.low[0]))
txt.append('{}'.format(self.data1.close[0]))
txt.append('{}'.format(self.data1.volume[0]))
txt.append('{}'.format(self.data1.openinterest[0]))
txt.append('{}'.format(float('NaN')))
print(', '.join(txt))
if self.counttostop: # stop after x live lines
self.counttostop -= 1
if not self.counttostop:
self.env.runstop()
return
if not self.p.trade:
return
if self.datastatus and not self.position and len(self.orderid) < 1:
self.order = self.buy(size=self.p.stake,
exectype=self.p.exectype,
price=round(self.data0.close[0] * 0.90, 2),
valid=self.p.valid)
self.orderid.append(self.order)
elif self.position.size > 0 and not self.p.donotsell:
if self.order is None:
self.order = self.sell(size=self.p.stake // 2,
exectype=bt.Order.Market,
price=self.data0.close[0])
elif self.order is not None and self.p.cancel:
if self.datastatus > self.p.cancel:
self.cancel(self.order)
if self.datastatus:
self.datastatus += 1
def start(self):
if self.data0.contractdetails is not None:
print('Timezone from ContractDetails: {}'.format(
self.data0.contractdetails.m_timeZoneId))
header = ['Datetime', 'Open', 'High', 'Low', 'Close', 'Volume',
'OpenInterest', 'SMA']
print(', '.join(header))
self.done = False
def runstrategy():
args = parse_args([
'--port', '7497',
'--data0', 'TWTR',
# '--data0', 'EUR.USD-CASH-IDEALPRO',
'--historical',
'--fromdate', '2018-01-02',
# '--no-backfill_start',
# '--no-backfill',
'--timeframe', bt.TimeFrame.Names[bt.TimeFrame.Days],
'--compression', '1',
'--timezone', 'NZ',
])
# Create a cerebro
cerebro = bt.Cerebro()
storekwargs = dict(
host=args.host, port=args.port,
clientId=args.clientId, timeoffset=not args.no_timeoffset,
reconnect=args.reconnect, timeout=args.timeout,
notifyall=args.notifyall, _debug=args.debug
)
if args.usestore:
ibstore = bt.stores.IBStore(**storekwargs)
if args.broker:
if args.usestore:
broker = ibstore.getbroker()
else:
broker = bt.brokers.IBBroker(**storekwargs)
cerebro.setbroker(broker)
timeframe = bt.TimeFrame.TFrame(args.timeframe)
if args.resample or args.replay:
datatf = bt.TimeFrame.Ticks
datacomp = 1
else:
datatf = timeframe
datacomp = args.compression
fromdate = None
if args.fromdate:
dtformat = '%Y-%m-%d' + ('T%H:%M:%S' * ('T' in args.fromdate))
fromdate = datetime.datetime.strptime(args.fromdate, dtformat)
IBDataFactory = ibstore.getdata if args.usestore else bt.feeds.IBData
datakwargs = dict(
timeframe=datatf, compression=datacomp,
historical=args.historical, fromdate=fromdate,
rtbar=args.rtbar,
qcheck=args.qcheck,
what=args.what,
backfill_start=not args.no_backfill_start,
backfill=not args.no_backfill,
latethrough=args.latethrough,
tz=args.timezone
)
if not args.usestore and not args.broker: # neither store nor broker
datakwargs.update(storekwargs) # pass the store args over the data
data0 = IBDataFactory(dataname=args.data0, **datakwargs)
data1 = None
if args.data1 is not None:
data1 = IBDataFactory(dataname=args.data1, **datakwargs)
rekwargs = dict(
timeframe=timeframe, compression=args.compression,
bar2edge=not args.no_bar2edge,
adjbartime=not args.no_adjbartime,
rightedge=not args.no_rightedge,
takelate=not args.no_takelate,
)
if args.replay:
cerebro.replaydata(dataname=data0, **rekwargs)
if data1 is not None:
cerebro.replaydata(dataname=data1, **rekwargs)
elif args.resample:
cerebro.resampledata(dataname=data0, **rekwargs)
if data1 is not None:
cerebro.resampledata(dataname=data1, **rekwargs)
else:
cerebro.adddata(data0)
if data1 is not None:
cerebro.adddata(data1)
if args.valid is None:
valid = None
else:
datetime.timedelta(seconds=args.valid)
# Add the strategy
cerebro.addstrategy(TestStrategy,
smaperiod=args.smaperiod,
trade=args.trade,
exectype=bt.Order.ExecType(args.exectype),
stake=args.stake,
stopafter=args.stopafter,
valid=valid,
cancel=args.cancel,
donotsell=args.donotsell)
# Live data ... avoid long data accumulation by switching to "exactbars"
cerebro.run(exactbars=args.exactbars)
if args.plot and args.exactbars < 1: # plot if possible
cerebro.plot()
def parse_args(pargs=None):
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description='Test Interactive Brokers integration')
parser.add_argument('--exactbars', default=1, type=int,
required=False, action='store',
help='exactbars level, use 0/-1/-2 to enable plotting')
parser.add_argument('--plot',
required=False, action='store_true',
help='Plot if possible')
parser.add_argument('--stopafter', default=0, type=int,
required=False, action='store',
help='Stop after x lines of LIVE data')
parser.add_argument('--usestore',
required=False, action='store_true',
help='Use the store pattern')
parser.add_argument('--notifyall',
required=False, action='store_true',
help='Notify all messages to strategy as store notifs')
parser.add_argument('--debug',
required=False, action='store_true',
help='Display all info received form IB')
parser.add_argument('--host', default='127.0.0.1',
required=False, action='store',
help='Host for the Interactive Brokers TWS Connection')
parser.add_argument('--qcheck', default=0.5, type=float,
required=False, action='store',
help=('Timeout for periodic '
'notification/resampling/replaying check'))
parser.add_argument('--port', default=7496, type=int,
required=False, action='store',
help='Port for the Interactive Brokers TWS Connection')
parser.add_argument('--clientId', default=None, type=int,
required=False, action='store',
help='Client Id to connect to TWS (default: random)')
parser.add_argument('--no-timeoffset',
required=False, action='store_true',
help=('Do not Use TWS/System time offset for non '
'timestamped prices and to align resampling'))
parser.add_argument('--reconnect', default=3, type=int,
required=False, action='store',
help='Number of recconnection attempts to TWS')
parser.add_argument('--timeout', default=3.0, type=float,
required=False, action='store',
help='Timeout between reconnection attempts to TWS')
parser.add_argument('--data0', default=None,
required=True, action='store',
help='data 0 into the system')
parser.add_argument('--data1', default=None,
required=False, action='store',
help='data 1 into the system')
parser.add_argument('--timezone', default=None,
required=False, action='store',
help='timezone to get time output into (pytz names)')
parser.add_argument('--what', default=None,
required=False, action='store',
help='specific price type for historical requests')
parser.add_argument('--no-backfill_start',
required=False, action='store_true',
help='Disable backfilling at the start')
parser.add_argument('--latethrough',
required=False, action='store_true',
help=('if resampling replaying, adjusting time '
'and disabling time offset, let late samples '
'through'))
parser.add_argument('--no-backfill',
required=False, action='store_true',
help='Disable backfilling after a disconnection')
parser.add_argument('--rtbar', default=False,
required=False, action='store_true',
help='Use 5 seconds real time bar updates if possible')
parser.add_argument('--historical',
required=False, action='store_true',
help='do only historical download')
parser.add_argument('--fromdate',
required=False, action='store',
help=('Starting date for historical download '
'with format: YYYY-MM-DD[THH:MM:SS]'))
parser.add_argument('--smaperiod', default=5, type=int,
required=False, action='store',
help='Period to apply to the Simple Moving Average')
pgroup = parser.add_mutually_exclusive_group(required=False)
pgroup.add_argument('--replay',
required=False, action='store_true',
help='replay to chosen timeframe')
pgroup.add_argument('--resample',
required=False, action='store_true',
help='resample to chosen timeframe')
parser.add_argument('--timeframe', default=bt.TimeFrame.Names[0],
choices=bt.TimeFrame.Names,
required=False, action='store',
help='TimeFrame for Resample/Replay')
parser.add_argument('--compression', default=1, type=int,
required=False, action='store',
help='Compression for Resample/Replay')
parser.add_argument('--no-takelate',
required=False, action='store_true',
help=('resample/replay, do not accept late samples '
'in new bar if the data source let them through '
'(latethrough)'))
parser.add_argument('--no-bar2edge',
required=False, action='store_true',
help='no bar2edge for resample/replay')
parser.add_argument('--no-adjbartime',
required=False, action='store_true',
help='no adjbartime for resample/replay')
parser.add_argument('--no-rightedge',
required=False, action='store_true',
help='no rightedge for resample/replay')
parser.add_argument('--broker',
required=False, action='store_true',
help='Use IB as broker')
parser.add_argument('--trade',
required=False, action='store_true',
help='Do Sample Buy/Sell operations')
parser.add_argument('--donotsell',
required=False, action='store_true',
help='Do not sell after a buy')
parser.add_argument('--exectype', default=bt.Order.ExecTypes[0],
choices=bt.Order.ExecTypes,
required=False, action='store',
help='Execution to Use when opening position')
parser.add_argument('--stake', default=10, type=int,
required=False, action='store',
help='Stake to use in buy operations')
parser.add_argument('--valid', default=None, type=int,
required=False, action='store',
help='Seconds to keep the order alive (0 means DAY)')
parser.add_argument('--cancel', default=0, type=int,
required=False, action='store',
help=('Cancel a buy order after n bars in operation,'
' to be combined with orders like Limit'))
if pargs is not None:
return parser.parse_args(pargs)
return parser.parse_args()
if __name__ == '__main__':
runstrategy()
Running this script, and the others that have been "shown" to work, results in:
Server Version: 76
TWS Time at connection:20200205 16:02:32 NZST
--------------------------------------------------
Strategy Created
--------------------------------------------------
Timezone from ContractDetails: EST (Eastern Standard Time)
Datetime, Open, High, Low, Close, Volume, OpenInterest, SMA
***** STORE NOTIF: <error id=-1, errorCode=2104, errorMsg=Market data farm connection is OK:cashfarm>
***** STORE NOTIF: <error id=-1, errorCode=2104, errorMsg=Market data farm connection is OK:usfarm>
***** STORE NOTIF: <error id=-1, errorCode=2106, errorMsg=HMDS data farm connection is OK:cashhmds>
***** STORE NOTIF: <error id=-1, errorCode=2106, errorMsg=HMDS data farm connection is OK:hkhmds>
***** STORE NOTIF: <error id=-1, errorCode=2106, errorMsg=HMDS data farm connection is OK:ushmds>
***** STORE NOTIF: <error id=-1, errorCode=2158, errorMsg=Sec-def data farm connection is OK:secdefhk>
***** DATA NOTIF: DELAYED
***** DATA NOTIF: DISCONNECTED
***** STORE NOTIF: <error id=16777217, errorCode=162, errorMsg=Historical Market Data Service error message:No market data permissions for NYSE STK>
Process finished with exit code 0
I just cannot for the life of me figure out why I cannot get any historical data - inside TWS (and the STORE NOTIFs above), all the data connections are working. I've tried following all of the sample scripts (even uninstalling and reinstalling TWS to "perfectly" follow along with the examples), and while those are able to get the data coming through, mine never seem too.
Does anybody have any piece of information that may help me solve this issue? I really don't know what to do from here...
Many thanks in advance!