For code/output blocks: Use ``` (aka backtick or grave accent) in a single line before and after the block. See: http://commonmark.org/help/

Assignement out of range error when running default strategy with pickle data



  • Hi all,

    I'm starting with backtrader and already encounter a beginner's error when
    trying to use my own data with one of the default strats.
    here are the first 10 lines of the data for illustration:

    open,high,low,close,volume,datetime,MARGIN
    -1.92,-1.9,-1.92,-1.92,24,2016-01-21 18:44:59.388700008,121
    -1.92,-1.92,-1.94,-1.94,8,2016-01-21 19:11:58.693599939,121
    -1.94,-1.91,-1.94,-1.91,12,2016-01-21 19:16:53.670099974,121
    -1.91,-1.91,-1.91,-1.91,10,2016-01-21 19:27:29.166599989,121
    -1.91,-1.91,-1.91,-1.91,10,2016-01-21 19:27:34.005199909,121
    -1.91,-1.91,-1.91,-1.91,10,2016-01-21 19:28:49.912800074,121
    -1.91,-1.88,-1.93,-1.88,11,2016-01-21 19:42:32.080300093,121
    -1.89,-1.89,-1.92,-1.92,9,2016-01-21 21:00:28.162600040,121
    -1.92,-1.89,-1.93,-1.89,22,2016-01-21 21:43:47.102799892,121
    -1.87,-1.85,-1.87,-1.85,8,2016-01-22 03:09:28.758999825,121
    

    I also put a bigger version of the data here.

    The strategy is the one I found here. I did not change it in any way except for using my own data above.

    from __future__ import (absolute_import, division, print_function,
                            unicode_literals)
    
    import datetime  # For datetime objects
    import os.path  # To manage paths
    import sys  # To find out the script name (in argv[0])
    
    # Import the backtrader platform
    import backtrader as bt
    import pandas as pd
    
    
    # Create a Stratey
    class TestStrategy(bt.Strategy):
        params = (
            ('maperiod', 15),
        )
    
        def log(self, txt, dt=None):
            ''' Logging function fot this strategy'''
            dt = dt or self.datas[0].datetime.date(0)
            print('%s, %s' % (dt.isoformat(), txt))
    
        def __init__(self):
            # Keep a reference to the "close" line in the data[0] dataseries
            self.dataclose = self.datas[0].close
    
            # To keep track of pending orders and buy price/commission
            self.order = None
            self.buyprice = None
            self.buycomm = None
    
            # Add a MovingAverageSimple indicator
            self.sma = bt.indicators.SimpleMovingAverage(
                self.datas[0], period=self.params.maperiod)
    
        def notify_order(self, order):
            if order.status in [order.Submitted, order.Accepted]:
                # Buy/Sell order submitted/accepted to/by broker - Nothing to do
                return
    
            # Check if an order has been completed
            # Attention: broker could reject order if not enough cash
            if order.status in [order.Completed]:
                if order.isbuy():
                    self.log(
                        'BUY EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' %
                        (order.executed.price,
                         order.executed.value,
                         order.executed.comm))
    
                    self.buyprice = order.executed.price
                    self.buycomm = order.executed.comm
                else:  # Sell
                    self.log('SELL EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' %
                             (order.executed.price,
                              order.executed.value,
                              order.executed.comm))
    
                self.bar_executed = len(self)
    
            elif order.status in [order.Canceled, order.Margin, order.Rejected]:
                self.log('Order Canceled/Margin/Rejected')
    
            self.order = None
    
        def notify_trade(self, trade):
            if not trade.isclosed:
                return
    
            self.log('OPERATION PROFIT, GROSS %.2f, NET %.2f' %
                     (trade.pnl, trade.pnlcomm))
    
        def next(self):
            # Simply log the closing price of the series from the reference
            self.log('Close, %.2f' % self.dataclose[0])
    
            # Check if an order is pending ... if yes, we cannot send a 2nd one
            if self.order:
                return
    
            # Check if we are in the market
            if not self.position:
    
                # Not yet ... we MIGHT BUY if ...
                if self.dataclose[0] > self.sma[0]:
    
                    # BUY, BUY, BUY!!! (with all possible default parameters)
                    self.log('BUY CREATE, %.2f' % self.dataclose[0])
    
                    # Keep track of the created order to avoid a 2nd order
                    self.order = self.buy()
    
            else:
    
                if self.dataclose[0] < self.sma[0]:
                    # SELL, SELL, SELL!!! (with all possible default parameters)
                    self.log('SELL CREATE, %.2f' % self.dataclose[0])
    
                    # Keep track of the created order to avoid a 2nd order
                    self.order = self.sell()
    
    
    class PandasData(bt.feed.DataBase):
        '''
        The ``dataname`` parameter inherited from ``feed.DataBase`` is the pandas
        DataFrame
        '''
    
        params = (
            # Possible values for datetime (must always be present)
            #  None : datetime is the "index" in the Pandas Dataframe
            #  -1 : autodetect position or case-wise equal name
            #  >= 0 : numeric index to the colum in the pandas dataframe
            #  string : column name (as index) in the pandas dataframe
            ('datetime', 5),
    
            # Possible values below:
            #  None : column not present
            #  -1 : autodetect position or case-wise equal name
            #  >= 0 : numeric index to the colum in the pandas dataframe
            #  string : column name (as index) in the pandas dataframe
            ('open', 0),
            ('high', 1),
            ('low', 2),
            ('close', 3),
            ('volume', 4),
            ('MARGIN', 4),
            ('openinterest', None),
        )
    
    
    
    if __name__ == '__main__':
    
        # Create a cerebro entity
        cerebro = bt.Cerebro()
        # Add a strategy
        cerebro.addstrategy(TestStrategy)
    
        # Create a Data Feed
        dataframe = pd.read_pickle('trial.pkl')
        data = PandasData(dataname=dataframe, timeframe = bt.TimeFrame.Ticks, openinterest = None)
    
        # Add the Data Feed to Cerebro
        cerebro.adddata(data)
    
        # Set our desired cash start
        cerebro.broker.setcash(100000.0)
        cerebro.broker.setcommission(commission=0.001)
    
        # Print out the starting conditions
        print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue())
    
        # Run over everything
        cerebro.run()
    
        # Print out the final result
        print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue())
    
    

    When I run this code, I get errors:

    IndexError                                Traceback (most recent call last)
    <ipython-input-10-c16d7ec32c11> in <module>()
         18
         19 # Run over everything
    ---> 20 cerebro.run()
         21
         22 # Print out the final result
    
    ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/cerebro.py in run(self, **kwargs)
       1125             # let's skip process "spawning"
       1126             for iterstrat in iterstrats:
    -> 1127                 runstrat = self.runstrategies(iterstrat)
       1128                 self.runstrats.append(runstrat)
       1129                 if self._dooptimize:
    
    ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/cerebro.py in runstrategies(self, iterstrat, predata)
       1291                     self._runonce_old(runstrats)
       1292                 else:
    -> 1293                     self._runonce(runstrats)
       1294             else:
       1295                 if self.p.oldsync:
    
    ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/cerebro.py in _runonce(self, runstrats)
       1650         '''
       1651         for strat in runstrats:
    -> 1652             strat._once()
       1653             strat.reset()  # strat called next by next - reset lines
       1654
    
    ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/lineiterator.py in _once(self)
        290
        291         for indicator in self._lineiterators[LineIterator.IndType]:
    --> 292             indicator._once()
        293
        294         for observer in self._lineiterators[LineIterator.ObsType]:
    
    ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/lineiterator.py in _once(self)
        290
        291         for indicator in self._lineiterators[LineIterator.IndType]:
    --> 292             indicator._once()
        293
        294         for observer in self._lineiterators[LineIterator.ObsType]:
    
    ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/lineiterator.py in _once(self)
        310         # indicators are each called with its min period
        311         self.preonce(0, self._minperiod - 1)
    --> 312         self.oncestart(self._minperiod - 1, self._minperiod)
        313         self.once(self._minperiod, self.buflen())
        314
    
    ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/lineiterator.py in oncestart(self, start, end)
        320
        321     def oncestart(self, start, end):
    --> 322         self.once(start, end)
        323
        324     def once(self, start, end):
    
    ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/indicators/basicops.py in once(self, start, end)
        362
        363         for i in range(start, end):
    --> 364             dst[i] = math.fsum(src[i - period + 1:i + 1]) / period
        365
        366
    
    IndexError: array assignment index out of range
    
    

    I suspect I am not importing my data correctly. I was wondering if a more seasoned user could point me in the correct direction@~


  • administrators

    @kav said in Assignement out of range error when running default strategy with pickle data:

    class PandasData(bt.feed.DataBase):
        '''
        The ``dataname`` parameter inherited from ``feed.DataBase`` is the pandas
        DataFrame
        '''
    
    

    You have created a data feed which has no functionality at all because you have not overridden any of the needed methods.

    • What is your expectation?
    • Why not using the existingPandasData feed?


  • @backtrader: Thank you for your answer. I had miss understood that part of the docs. I have a related question though.

    So first, I fix my bodged attempt:

    from __future__ import (absolute_import, division, print_function,
                            unicode_literals)
    
    import datetime  # For datetime objects
    import os.path  # To manage paths
    import sys  # To find out the script name (in argv[0])
    
    # Import the backtrader platform
    import backtrader as bt
    import pandas as pd
    
    
    # Create a Stratey
    class maCross(bt.Strategy):
        '''
        For an official backtrader blog on this topic please take a look at:
    
        https://www.backtrader.com/blog/posts/2017-04-09-multi-example/multi-example.html
    
        oneplot = Force all datas to plot on the same master.
        '''
        params = (
        ('sma1', 80),
        ('sma2', 200),
        ('oneplot', True),
        ('list_plus', [])
        )
    
        def __init__(self):
            '''
            Create an dictionary of indicators so that we can dynamically add the
            indicators to the strategy using a loop. This mean the strategy will
            work with any number of data feeds. 
            '''
            self.inds = dict()#           p0 = pandas.Series(self.data.close.get(size=5))
    #           print(p0.head(5).transpose())
            for i, d in enumerate(self.datas):
                self.inds[d] = dict()
                self.inds[d]['sma1'] = bt.indicators.SMA(d.close, 
                                                                 period=self.params.sma1)
                self.inds[d]['sma2'] = bt.indicators.SMA(d.close, 
                                                                 period=self.params.sma2)
                if d._name in self.params.list_plus:
                    self.inds[d]['cross'] = bt.indicators.CrossOver(self.inds[d]['sma1'],
                                                                            self.inds[d]['sma2'])
                else:
                    self.inds[d]['cross'] = bt.indicators.CrossOver(self.inds[d]['sma2'],
                                                                            self.inds[d]['sma1']) 
              
                if i > 0: #Check we are not on the first loop of data feed:
                    if self.p.oneplot == True:
                        d.plotinfo.plotmaster = self.datas[0]
    
        def next(self):
            for i, d in enumerate(self.datas):
                dt, dn = self.datetime.date(), d._name
                pos = self.getposition(d).size
    
                if not pos:  # no market / no orders
                    if self.inds[d]['cross'][0] == 1:
                        self.buy(data=d, size=1000)
                    elif self.inds[d]['cross'][0] == -1:
                        self.sell(data=d, size=1000)
                else:
                    if self.inds[d]['cross'][0] == 1:
                        self.close(data=d)
                        self.buy(data=d, size=1000)
                    elif self.inds[d]['cross'][0] == -1:
                        self.close(data=d)
                        self.sell(data=d, size=1000)
            
        def notify_trade(self, trade):
            dt = self.data.datetime.date()
            if trade.isclosed:
                print('{} {} Closed: PnL Gross {}, Net {}'.format(
                                                    dt,
                                                    trade.data._name,
                                                    round(trade.pnl,2),
                                                    round(trade.pnlcomm,2)))
    
    
    #if __name__ == '__main__':
    if 1==0:
        # Create a cerebro entity
        cerebro = bt.Cerebro()
        # Add a strategy
        cerebro.addstrategy(maCross)
    
        # Create a Data Feed
        dataframe = pd.read_pickle('trial.pkl')
        data = bt.feeds.PandasData(dataname = dataframe, 
                                   datetime = 5, 
                                   open     = 0, 
                                   high     = 1, 
                                   low      = 2, 
                                   close    = 3,
                                   volume   = 4,
                                   openinterest = None)
    
    
        # Add the Data Feed to Cerebro
        cerebro.adddata(data)
    
        # Set our desired cash start
        cerebro.broker.setcash(100000.0)
        cerebro.broker.setcommission(commission=0.001)
    
        # Print out the starting conditions
        print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue())
    
        # Run over everything
        cerebro.run()
    
        # Print out the final result
        print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue())
    

    (e.g. not overloading PandasData).

    There is one thing I still do not seem to get working tho. Using the code above, I get:

    ---------------------------------------------------------------------------
    AttributeError                            Traceback (most recent call last)
    <ipython-input-15-c302340a2d92> in <module>()
    ----> 1 cerebro.run()
    
    ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/cerebro.py in run(self, **kwargs)
       1125             # let's skip process "spawning"
       1126             for iterstrat in iterstrats:
    -> 1127                 runstrat = self.runstrategies(iterstrat)
       1128                 self.runstrats.append(runstrat)
       1129                 if self._dooptimize:
    
    ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/cerebro.py in runstrategies(self, iterstrat, predata)
       1210                 data._start()
       1211                 if self._dopreload:
    -> 1212                     data.preload()
       1213
       1214         for stratcls, sargs, skwargs in iterstrat:
    
    ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/feed.py in preload(self)
        436
        437     def preload(self):
    --> 438         while self.load():
        439             pass
        440
    
    ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/feed.py in load(self)
        477
        478             if not self._fromstack(stash=True):
    --> 479                 _loadret = self._load()
        480                 if not _loadret:  # no bar use force to make sure in exactbars
        481                     # the pointer is undone this covers especially (but not
    
    ~/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/feeds/pandafeed.py in _load(self)
        266
        267         # convert to float via datetime and store it
    --> 268         dt = tstamp.to_pydatetime()
        269         dtnum = date2num(dt)
        270         self.lines.datetime[0] = dtnum
    
    AttributeError: 'numpy.float64' object has no attribute 'to_pydatetime'
    

    I think this is related to my not having imported the datetimeindex right (?).

    FWIW , pd.to_datetime(dataframe.datetime, unit = 's') seems to work;~ I have also followed in adding a timeframe = bt.TimeFrame.Ticks to the bt.feeds.PandasData( call, but that doesn't solve the problem.


  • administrators

    @kav said in Assignement out of range error when running default strategy with pickle data:

    I think this is related to my not having imported the datetimeindex right (?).

    There is a float where you are telling the framework to find a datetime. When using pandas you would usually use, for example read_csv with parse_dates=True

    @kav said in Assignement out of range error when running default strategy with pickle data:

    FWIW , pd.to_datetime(dataframe.datetime, unit = 's') seems to work;

    Because you are transforming the float into a datetime

    @kav said in Assignement out of range error when running default strategy with pickle data:

    I have also followed in adding a timeframe = bt.TimeFrame.Ticks to the bt.feeds.PandasData( call, but that doesn't solve the problem.

    Telling the framework which timeframe your data has is not going to make the conversion. You need it to let the platform know in which context things are (for example for later resampling to a higher timeframe)



  • @backtrader: thank you. With your help, I think it now works. What I did was to change:

    dataframe = pd.read_pickle('trial.pkl')
    

    to:

    dataframe = pd.read_pickle('trial.pkl').set_index('datetime')
    

    and the data import to:

        data = bt.feeds.PandasData(dataname = dataframe, 
                                   datetime = -1, 
                                   open     = 0, 
                                   high     = 1, 
                                   low      = 2, 
                                   close    = 3,
                                   volume   = 4,
                                   openinterest = None)
    

    (full code for reference at the end).
    and it seems to work:

    Starting Portfolio Value: 100000.00
    /home/kaveh/Desktop/work/p1/geqw4/vi3/out/sp/ezalgo/Strats/lib/python3.6/site-packages/backtrader/feed.py:479: UserWarning: Discarding nonzero nanoseconds in conversion
      _loadret = self._load()
    2016-01-29  Closed: PnL Gross 150.0, Net 154.33
    Out[11]: [<__main__.maCross at 0x7fda173ba908>]
    
    In [12]: print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue())
    Final Portfolio Value: 100076.57
    

    Full code:

    from __future__ import (absolute_import, division, print_function,
                            unicode_literals)
    
    import datetime  # For datetime objects
    import os.path  # To manage paths
    import sys  # To find out the script name (in argv[0])
    
    # Import the backtrader platform
    import backtrader as bt
    import pandas as pd
    
    
    # Create a Stratey
    class maCross(bt.Strategy):
        '''
        For an official backtrader blog on this topic please take a look at:
    
        https://www.backtrader.com/blog/posts/2017-04-09-multi-example/multi-example.html
    
        oneplot = Force all datas to plot on the same master.
        '''
        params = (
        ('sma1', 80),
        ('sma2', 200),
        ('oneplot', True),
        ('list_plus', [])
        )
    
        def __init__(self):
            '''
            Create an dictionary of indicators so that we can dynamically add the
            indicators to the strategy using a loop. This mean the strategy will
            work with any number of data feeds. 
            '''
            self.inds = dict()#           p0 = pandas.Series(self.data.close.get(size=5))
    #           print(p0.head(5).transpose())
            for i, d in enumerate(self.datas):
                self.inds[d] = dict()
                self.inds[d]['sma1'] = bt.indicators.SMA(d.close, 
                                                                 period=self.params.sma1)
                self.inds[d]['sma2'] = bt.indicators.SMA(d.close, 
                                                                 period=self.params.sma2)
                if d._name in self.params.list_plus:
                    self.inds[d]['cross'] = bt.indicators.CrossOver(self.inds[d]['sma1'],
                                                                            self.inds[d]['sma2'])
                else:
                    self.inds[d]['cross'] = bt.indicators.CrossOver(self.inds[d]['sma2'],
                                                                            self.inds[d]['sma1']) 
              
                if i > 0: #Check we are not on the first loop of data feed:
                    if self.p.oneplot == True:
                        d.plotinfo.plotmaster = self.datas[0]
    
        def next(self):
            for i, d in enumerate(self.datas):
                dt, dn = self.datetime.date(), d._name
                pos = self.getposition(d).size
    
                if not pos:  # no market / no orders
                    if self.inds[d]['cross'][0] == 1:
                        self.buy(data=d, size=1000)
                    elif self.inds[d]['cross'][0] == -1:
                        self.sell(data=d, size=1000)
                else:
                    if self.inds[d]['cross'][0] == 1:
                        self.close(data=d)
                        self.buy(data=d, size=1000)
                    elif self.inds[d]['cross'][0] == -1:
                        self.close(data=d)
                        self.sell(data=d, size=1000)
            
        def notify_trade(self, trade):
            dt = self.data.datetime.date()
            if trade.isclosed:
                print('{} {} Closed: PnL Gross {}, Net {}'.format(
                                                    dt,
                                                    trade.data._name,
                                                    round(trade.pnl,2),
                                                    round(trade.pnlcomm,2)))
    
    
    if __name__ == '__main__':
    #if 1==0:
        # Create a cerebro entity
        cerebro = bt.Cerebro()
        # Add a strategy
        cerebro.addstrategy(maCross)
    
        # Create a Data Feed
        dataframe = pd.read_pickle('trial.pkl').set_index('datetime')
        # dataframe.drop('datetime', axis = 1, inplace = True)
        # dataframe.reset_index(inplace = True)
        # dataframe.rename({'ACTIVITY_DATETIME':'datetime'}, copy = False, inplace = True, axis = 1)
        data = bt.feeds.PandasData(dataname = dataframe, 
                                   datetime = -1, 
                                   open     = 0, 
                                   high     = 1, 
                                   low      = 2, 
                                   close    = 3,
                                   volume   = 4,
                                   openinterest = None)
    
    
        # Add the Data Feed to Cerebro
        cerebro.adddata(data)
    
        # Set our desired cash start
        cerebro.broker.setcash(100000.0)
        cerebro.broker.setcommission(commission=0.001)
    
        # Print out the starting conditions
        print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue())
    
        # Run over everything
        cerebro.run()
    
        # Print out the final result
        print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue())
    

  • administrators

    @kav said in Assignement out of range error when running default strategy with pickle data:

    For an official backtrader blog on this topic please take a look at:
    
    https://www.bt.com/blog/posts/2017-04-09-multi-example/multi-example.html
    

    I don't think that www.bt.com is hosting content related to backtrader ...



  • ah:) @backtrader : good catch. It's probably a CTRL-H gone awry. I fixed it in the codes above. Thanks!


Log in to reply
 

});