For code/output blocks: Use ``` (aka backtick or grave accent) in a single line before and after the block. See: http://commonmark.org/help/
Assignement out of range error when running default strategy with pickle data
-
Hi all,
I'm starting with backtrader and already encounter a beginner's error when
trying to use my own data with one of the default strats.
here are the first 10 lines of the data for illustration:open,high,low,close,volume,datetime,MARGIN -1.92,-1.9,-1.92,-1.92,24,2016-01-21 18:44:59.388700008,121 -1.92,-1.92,-1.94,-1.94,8,2016-01-21 19:11:58.693599939,121 -1.94,-1.91,-1.94,-1.91,12,2016-01-21 19:16:53.670099974,121 -1.91,-1.91,-1.91,-1.91,10,2016-01-21 19:27:29.166599989,121 -1.91,-1.91,-1.91,-1.91,10,2016-01-21 19:27:34.005199909,121 -1.91,-1.91,-1.91,-1.91,10,2016-01-21 19:28:49.912800074,121 -1.91,-1.88,-1.93,-1.88,11,2016-01-21 19:42:32.080300093,121 -1.89,-1.89,-1.92,-1.92,9,2016-01-21 21:00:28.162600040,121 -1.92,-1.89,-1.93,-1.89,22,2016-01-21 21:43:47.102799892,121 -1.87,-1.85,-1.87,-1.85,8,2016-01-22 03:09:28.758999825,121I also put a bigger version of the data here.
The strategy is the one I found here. I did not change it in any way except for using my own data above.
from __future__ import (absolute_import, division, print_function, unicode_literals) import datetime # For datetime objects import os.path # To manage paths import sys # To find out the script name (in argv[0]) # Import the backtrader platform import backtrader as bt import pandas as pd # Create a Stratey class TestStrategy(bt.Strategy): params = ( ('maperiod', 15), ) def log(self, txt, dt=None): ''' Logging function fot this strategy''' dt = dt or self.datas[0].datetime.date(0) print('%s, %s' % (dt.isoformat(), txt)) def __init__(self): # Keep a reference to the "close" line in the data[0] dataseries self.dataclose = self.datas[0].close # To keep track of pending orders and buy price/commission self.order = None self.buyprice = None self.buycomm = None # Add a MovingAverageSimple indicator self.sma = bt.indicators.SimpleMovingAverage( self.datas[0], period=self.params.maperiod) def notify_order(self, order): if order.status in [order.Submitted, order.Accepted]: # Buy/Sell order submitted/accepted to/by broker - Nothing to do return # Check if an order has been completed # Attention: broker could reject order if not enough cash if order.status in [order.Completed]: if order.isbuy(): self.log( 'BUY EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' % (order.executed.price, order.executed.value, order.executed.comm)) self.buyprice = order.executed.price self.buycomm = order.executed.comm else: # Sell self.log('SELL EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' % (order.executed.price, order.executed.value, order.executed.comm)) self.bar_executed = len(self) elif order.status in [order.Canceled, order.Margin, order.Rejected]: self.log('Order Canceled/Margin/Rejected') self.order = None def notify_trade(self, trade): if not trade.isclosed: return self.log('OPERATION PROFIT, GROSS %.2f, NET %.2f' % (trade.pnl, trade.pnlcomm)) def next(self): # Simply log the closing price of the series from the reference self.log('Close, %.2f' % self.dataclose[0]) # Check if an order is pending ... if yes, we cannot send a 2nd one if self.order: return # Check if we are in the market if not self.position: # Not yet ... we MIGHT BUY if ... if self.dataclose[0] > self.sma[0]: # BUY, BUY, BUY!!! (with all possible default parameters) self.log('BUY CREATE, %.2f' % self.dataclose[0]) # Keep track of the created order to avoid a 2nd order self.order = self.buy() else: if self.dataclose[0] < self.sma[0]: # SELL, SELL, SELL!!! (with all possible default parameters) self.log('SELL CREATE, %.2f' % self.dataclose[0]) # Keep track of the created order to avoid a 2nd order self.order = self.sell() class PandasData(bt.feed.DataBase): ''' The ``dataname`` parameter inherited from ``feed.DataBase`` is the pandas DataFrame ''' params = ( # Possible values for datetime (must always be present) # None : datetime is the "index" in the Pandas Dataframe # -1 : autodetect position or case-wise equal name # >= 0 : numeric index to the colum in the pandas dataframe # string : column name (as index) in the pandas dataframe ('datetime', 5), # Possible values below: # None : column not present # -1 : autodetect position or case-wise equal name # >= 0 : numeric index to the colum in the pandas dataframe # string : column name (as index) in the pandas dataframe ('open', 0), ('high', 1), ('low', 2), ('close', 3), ('volume', 4), ('MARGIN', 4), ('openinterest', None), ) if __name__ == '__main__': # Create a cerebro entity cerebro = bt.Cerebro() # Add a strategy cerebro.addstrategy(TestStrategy) # Create a Data Feed dataframe = pd.read_pickle('trial.pkl') data = PandasData(dataname=dataframe, timeframe = bt.TimeFrame.Ticks, openinterest = None) # Add the Data Feed to Cerebro cerebro.adddata(data) # Set our desired cash start cerebro.broker.setcash(100000.0) cerebro.broker.setcommission(commission=0.001) # Print out the starting conditions print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue()) # Run over everything cerebro.run() # Print out the final result print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue())When I run this code, I get errors:
IndexError Traceback (most recent call last) <ipython-input-10-c16d7ec32c11> in <module>() 18 19 # Run over everything ---> 20 cerebro.run() 21 22 # Print out the final result ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/cerebro.py in run(self, **kwargs) 1125 # let's skip process "spawning" 1126 for iterstrat in iterstrats: -> 1127 runstrat = self.runstrategies(iterstrat) 1128 self.runstrats.append(runstrat) 1129 if self._dooptimize: ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/cerebro.py in runstrategies(self, iterstrat, predata) 1291 self._runonce_old(runstrats) 1292 else: -> 1293 self._runonce(runstrats) 1294 else: 1295 if self.p.oldsync: ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/cerebro.py in _runonce(self, runstrats) 1650 ''' 1651 for strat in runstrats: -> 1652 strat._once() 1653 strat.reset() # strat called next by next - reset lines 1654 ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/lineiterator.py in _once(self) 290 291 for indicator in self._lineiterators[LineIterator.IndType]: --> 292 indicator._once() 293 294 for observer in self._lineiterators[LineIterator.ObsType]: ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/lineiterator.py in _once(self) 290 291 for indicator in self._lineiterators[LineIterator.IndType]: --> 292 indicator._once() 293 294 for observer in self._lineiterators[LineIterator.ObsType]: ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/lineiterator.py in _once(self) 310 # indicators are each called with its min period 311 self.preonce(0, self._minperiod - 1) --> 312 self.oncestart(self._minperiod - 1, self._minperiod) 313 self.once(self._minperiod, self.buflen()) 314 ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/lineiterator.py in oncestart(self, start, end) 320 321 def oncestart(self, start, end): --> 322 self.once(start, end) 323 324 def once(self, start, end): ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/indicators/basicops.py in once(self, start, end) 362 363 for i in range(start, end): --> 364 dst[i] = math.fsum(src[i - period + 1:i + 1]) / period 365 366 IndexError: array assignment index out of rangeI suspect I am not importing my data correctly. I was wondering if a more seasoned user could point me in the correct direction@~
-
@kav said in Assignement out of range error when running default strategy with pickle data:
class PandasData(bt.feed.DataBase): ''' The ``dataname`` parameter inherited from ``feed.DataBase`` is the pandas DataFrame '''You have created a data feed which has no functionality at all because you have not overridden any of the needed methods.
- What is your expectation?
- Why not using the existing
PandasDatafeed?
-
@backtrader: Thank you for your answer. I had miss understood that part of the docs. I have a related question though.
So first, I fix my bodged attempt:
from __future__ import (absolute_import, division, print_function, unicode_literals) import datetime # For datetime objects import os.path # To manage paths import sys # To find out the script name (in argv[0]) # Import the backtrader platform import backtrader as bt import pandas as pd # Create a Stratey class maCross(bt.Strategy): ''' For an official backtrader blog on this topic please take a look at: https://www.backtrader.com/blog/posts/2017-04-09-multi-example/multi-example.html oneplot = Force all datas to plot on the same master. ''' params = ( ('sma1', 80), ('sma2', 200), ('oneplot', True), ('list_plus', []) ) def __init__(self): ''' Create an dictionary of indicators so that we can dynamically add the indicators to the strategy using a loop. This mean the strategy will work with any number of data feeds. ''' self.inds = dict()# p0 = pandas.Series(self.data.close.get(size=5)) # print(p0.head(5).transpose()) for i, d in enumerate(self.datas): self.inds[d] = dict() self.inds[d]['sma1'] = bt.indicators.SMA(d.close, period=self.params.sma1) self.inds[d]['sma2'] = bt.indicators.SMA(d.close, period=self.params.sma2) if d._name in self.params.list_plus: self.inds[d]['cross'] = bt.indicators.CrossOver(self.inds[d]['sma1'], self.inds[d]['sma2']) else: self.inds[d]['cross'] = bt.indicators.CrossOver(self.inds[d]['sma2'], self.inds[d]['sma1']) if i > 0: #Check we are not on the first loop of data feed: if self.p.oneplot == True: d.plotinfo.plotmaster = self.datas[0] def next(self): for i, d in enumerate(self.datas): dt, dn = self.datetime.date(), d._name pos = self.getposition(d).size if not pos: # no market / no orders if self.inds[d]['cross'][0] == 1: self.buy(data=d, size=1000) elif self.inds[d]['cross'][0] == -1: self.sell(data=d, size=1000) else: if self.inds[d]['cross'][0] == 1: self.close(data=d) self.buy(data=d, size=1000) elif self.inds[d]['cross'][0] == -1: self.close(data=d) self.sell(data=d, size=1000) def notify_trade(self, trade): dt = self.data.datetime.date() if trade.isclosed: print('{} {} Closed: PnL Gross {}, Net {}'.format( dt, trade.data._name, round(trade.pnl,2), round(trade.pnlcomm,2))) #if __name__ == '__main__': if 1==0: # Create a cerebro entity cerebro = bt.Cerebro() # Add a strategy cerebro.addstrategy(maCross) # Create a Data Feed dataframe = pd.read_pickle('trial.pkl') data = bt.feeds.PandasData(dataname = dataframe, datetime = 5, open = 0, high = 1, low = 2, close = 3, volume = 4, openinterest = None) # Add the Data Feed to Cerebro cerebro.adddata(data) # Set our desired cash start cerebro.broker.setcash(100000.0) cerebro.broker.setcommission(commission=0.001) # Print out the starting conditions print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue()) # Run over everything cerebro.run() # Print out the final result print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue())(e.g. not overloading
PandasData).There is one thing I still do not seem to get working tho. Using the code above, I get:
--------------------------------------------------------------------------- AttributeError Traceback (most recent call last) <ipython-input-15-c302340a2d92> in <module>() ----> 1 cerebro.run() ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/cerebro.py in run(self, **kwargs) 1125 # let's skip process "spawning" 1126 for iterstrat in iterstrats: -> 1127 runstrat = self.runstrategies(iterstrat) 1128 self.runstrats.append(runstrat) 1129 if self._dooptimize: ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/cerebro.py in runstrategies(self, iterstrat, predata) 1210 data._start() 1211 if self._dopreload: -> 1212 data.preload() 1213 1214 for stratcls, sargs, skwargs in iterstrat: ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/feed.py in preload(self) 436 437 def preload(self): --> 438 while self.load(): 439 pass 440 ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/feed.py in load(self) 477 478 if not self._fromstack(stash=True): --> 479 _loadret = self._load() 480 if not _loadret: # no bar use force to make sure in exactbars 481 # the pointer is undone this covers especially (but not ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/feeds/pandafeed.py in _load(self) 266 267 # convert to float via datetime and store it --> 268 dt = tstamp.to_pydatetime() 269 dtnum = date2num(dt) 270 self.lines.datetime[0] = dtnum AttributeError: 'numpy.float64' object has no attribute 'to_pydatetime'I think this is related to my not having imported the datetimeindex right (?).
FWIW ,
pd.to_datetime(dataframe.datetime, unit = 's')seems to work;~ I have also followed in adding atimeframe = bt.TimeFrame.Ticksto thebt.feeds.PandasData(call, but that doesn't solve the problem. -
@kav said in Assignement out of range error when running default strategy with pickle data:
I think this is related to my not having imported the datetimeindex right (?).
There is a
floatwhere you are telling the framework to find adatetime. When usingpandasyou would usually use, for exampleread_csvwithparse_dates=True@kav said in Assignement out of range error when running default strategy with pickle data:
FWIW , pd.to_datetime(dataframe.datetime, unit = 's') seems to work;
Because you are transforming the
floatinto adatetime@kav said in Assignement out of range error when running default strategy with pickle data:
I have also followed in adding a timeframe = bt.TimeFrame.Ticks to the bt.feeds.PandasData( call, but that doesn't solve the problem.
Telling the framework which timeframe your data has is not going to make the conversion. You need it to let the platform know in which context things are (for example for later resampling to a higher timeframe)
-
@backtrader: thank you. With your help, I think it now works. What I did was to change:
dataframe = pd.read_pickle('trial.pkl')to:
dataframe = pd.read_pickle('trial.pkl').set_index('datetime')and the data import to:
data = bt.feeds.PandasData(dataname = dataframe, datetime = -1, open = 0, high = 1, low = 2, close = 3, volume = 4, openinterest = None)(full code for reference at the end).
and it seems to work:Starting Portfolio Value: 100000.00 /home/kaveh/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/feed.py:479: UserWarning: Discarding nonzero nanoseconds in conversion _loadret = self._load() 2016-01-29 Closed: PnL Gross 150.0, Net 154.33 Out[11]: [<__main__.maCross at 0x7fda173ba908>] In [12]: print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue()) Final Portfolio Value: 100076.57Full code:
from __future__ import (absolute_import, division, print_function, unicode_literals) import datetime # For datetime objects import os.path # To manage paths import sys # To find out the script name (in argv[0]) # Import the backtrader platform import backtrader as bt import pandas as pd # Create a Stratey class maCross(bt.Strategy): ''' For an official backtrader blog on this topic please take a look at: https://www.backtrader.com/blog/posts/2017-04-09-multi-example/multi-example.html oneplot = Force all datas to plot on the same master. ''' params = ( ('sma1', 80), ('sma2', 200), ('oneplot', True), ('list_plus', []) ) def __init__(self): ''' Create an dictionary of indicators so that we can dynamically add the indicators to the strategy using a loop. This mean the strategy will work with any number of data feeds. ''' self.inds = dict()# p0 = pandas.Series(self.data.close.get(size=5)) # print(p0.head(5).transpose()) for i, d in enumerate(self.datas): self.inds[d] = dict() self.inds[d]['sma1'] = bt.indicators.SMA(d.close, period=self.params.sma1) self.inds[d]['sma2'] = bt.indicators.SMA(d.close, period=self.params.sma2) if d._name in self.params.list_plus: self.inds[d]['cross'] = bt.indicators.CrossOver(self.inds[d]['sma1'], self.inds[d]['sma2']) else: self.inds[d]['cross'] = bt.indicators.CrossOver(self.inds[d]['sma2'], self.inds[d]['sma1']) if i > 0: #Check we are not on the first loop of data feed: if self.p.oneplot == True: d.plotinfo.plotmaster = self.datas[0] def next(self): for i, d in enumerate(self.datas): dt, dn = self.datetime.date(), d._name pos = self.getposition(d).size if not pos: # no market / no orders if self.inds[d]['cross'][0] == 1: self.buy(data=d, size=1000) elif self.inds[d]['cross'][0] == -1: self.sell(data=d, size=1000) else: if self.inds[d]['cross'][0] == 1: self.close(data=d) self.buy(data=d, size=1000) elif self.inds[d]['cross'][0] == -1: self.close(data=d) self.sell(data=d, size=1000) def notify_trade(self, trade): dt = self.data.datetime.date() if trade.isclosed: print('{} {} Closed: PnL Gross {}, Net {}'.format( dt, trade.data._name, round(trade.pnl,2), round(trade.pnlcomm,2))) if __name__ == '__main__': #if 1==0: # Create a cerebro entity cerebro = bt.Cerebro() # Add a strategy cerebro.addstrategy(maCross) # Create a Data Feed dataframe = pd.read_pickle('trial.pkl').set_index('datetime') # dataframe.drop('datetime', axis = 1, inplace = True) # dataframe.reset_index(inplace = True) # dataframe.rename({'ACTIVITY_DATETIME':'datetime'}, copy = False, inplace = True, axis = 1) data = bt.feeds.PandasData(dataname = dataframe, datetime = -1, open = 0, high = 1, low = 2, close = 3, volume = 4, openinterest = None) # Add the Data Feed to Cerebro cerebro.adddata(data) # Set our desired cash start cerebro.broker.setcash(100000.0) cerebro.broker.setcommission(commission=0.001) # Print out the starting conditions print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue()) # Run over everything cerebro.run() # Print out the final result print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue()) -
@kav said in Assignement out of range error when running default strategy with pickle data:
For an official backtrader blog on this topic please take a look at: https://www.bt.com/blog/posts/2017-04-09-multi-example/multi-example.htmlI don't think that
www.bt.comis hosting content related to backtrader ... -
ah:) @backtrader : good catch. It's probably a CTRL-H gone awry. I fixed it in the codes above. Thanks!