For code/output blocks: Use ``` (aka backtick or grave accent) in a single line before and after the block. See: http://commonmark.org/help/

Getting IndexError: array assignment index out of range in cerebro.run()



  • Hi everyone,

    I am a backtrader newbie and couldn't solve this problem. I am getting json data from mongodb and converting it into Pandas DataFrame and feeding it into cerebro.

    Here is the example data I have

                 datetime    open    high     low   close    volume
    0 2019-05-04 14:00:00  159.81  160.55  158.85  160.00   5281.80
    1 2019-05-04 13:00:00  161.93  162.19  158.00  159.81  12089.68
    2 2019-05-04 12:00:00  162.05  162.27  161.27  162.02   3296.24
    3 2019-05-04 11:00:00  162.25  162.41  161.90  162.01   7119.30
    4 2019-05-04 10:00:00  162.55  162.55  161.45  162.26   4599.58
    

    The helloworld example for backtrader can be seen below;

    from datetime import datetime
    import backtrader as bt
    from mongoengine import connect
    from pymongo import MongoClient
    import pandas as pd
    
    class TestStrategy(bt.Strategy):
        def log(self, txt, dt=None):
            dt = dt or self.datas[0].datetime.date(0)
            print('%s, %s' % (dt.isoformat(), txt))
    
        def __init__(self):
            self.dataclose = self.datas[0].close
            self.order = None
            self.buyprice = None
            self.buycomm = None
    
            self.sma = bt.indicators.SimpleMovingAverage(self.datas[0], period=15)
            self.rsi = bt.indicators.RelativeStrengthIndex()
    
        def notify_order(self, order):
            if order.status in [order.Submitted, order.Accepted]:
                return
    
            if order.status in [order.Completed]:
                if order.isbuy():
                    self.log(
                        'BUY EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' %
                        (order.executed.price,
                         order.executed.value,
                         order.executed.comm))
    
                    self.buyprice = order.executed.price
                    self.buycomm = order.executed.comm
                else:  # Sell
                    self.log('SELL EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' %
                             (order.executed.price,
                              order.executed.value,
                              order.executed.comm))
    
                self.bar_executed = len(self)
    
            elif order.status in [order.Canceled, order.Margin, order.Rejected]:
                self.log('Order Canceled/Margin/Rejected')
    
            # Write down: no pending order
            self.order = None
    
        def notify_trade(self, trade):
            if not trade.isclosed:
                return
    
            self.log('OPERATION PROFIT, GROSS %.2f, NET %.2f' %
                     (trade.pnl, trade.pnlcomm))
    
        def next(self):
            self.log('Close, %.2f' % self.dataclose[0])
            print('rsi:', self.rsi[0])
            if self.order:
                return
    
            if not self.position:
                if (self.rsi[0] < 30):
                    self.log('BUY CREATE, %.2f' % self.dataclose[0])
                    self.order = self.buy(size=500)
    
            else:
                if (self.rsi[0] > 70):
                    self.log('SELL CREATE, %.2f' % self.dataclose[0])
                    self.order = self.sell(size=500)
    
    
    class PandasData(bt.feed.DataBase):
        '''
        The ``dataname`` parameter inherited from ``feed.DataBase`` is the pandas
        DataFrame
        '''
    
        params = (
            # Possible values for datetime (must always be present)
            #  None : datetime is the "index" in the Pandas Dataframe
            #  -1 : autodetect position or case-wise equal name
            #  >= 0 : numeric index to the colum in the pandas dataframe
            #  string : column name (as index) in the pandas dataframe
            ('datetime', 0),
    
            # Possible values below:
            #  None : column not present
            #  -1 : autodetect position or case-wise equal name
            #  >= 0 : numeric index to the colum in the pandas dataframe
            #  string : column name (as index) in the pandas dataframe
            ('open', 1),
            ('high', 2),
            ('low', 3),
            ('close', 4),
            ('volume', 5),
            ('openinterest', None),
        )
    
    
    if __name__ == '__main__':
        cerebro = bt.Cerebro()
        cerebro.addstrategy(TestStrategy)
        cerebro.broker.setcommission(commission=0.001)
    
        connect('algotrader')
        client = MongoClient()
        db = client.get_database('algotrader')
        candles = db.get_collection('candles')
        candle_data = candles.find().sort([('time', -1)])
    
        ohlc = []
        for candle in candle_data:
            ohlc.append([datetime.utcfromtimestamp(candle['time']),
                             candle['open'],
                             candle['high'],
                             candle['low'],
                             candle['close'],
                             candle['volume']])
    
        df = pd.DataFrame(ohlc, columns = ['datetime', 'open', 'high', 'low', 'close', 'volume'])
        data = PandasData(dataname=df,
                          datetime=0,
                          open=1,
                          high=2,
                          low=3,
                          close=4,
                          volume=5)
        cerebro.adddata(data)
        cerebro.broker.setcash(1000.0)
        print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue())
        cerebro.run()
        print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue())
        cerebro.plot()
    

    Finally, here is the error I am getting;

    Traceback (most recent call last):
      File "backtesting.py", line 135, in <module>
        cerebro.run()
      File "/Users/buraktas/.virtualenvs/alga-trader/lib/python3.7/site-packages/backtrader/cerebro.py", line 1127, in run
        runstrat = self.runstrategies(iterstrat)
      File "/Users/buraktas/.virtualenvs/alga-trader/lib/python3.7/site-packages/backtrader/cerebro.py", line 1293, in runstrategies
        self._runonce(runstrats)
      File "/Users/buraktas/.virtualenvs/alga-trader/lib/python3.7/site-packages/backtrader/cerebro.py", line 1652, in _runonce
        strat._once()
      File "/Users/buraktas/.virtualenvs/alga-trader/lib/python3.7/site-packages/backtrader/lineiterator.py", line 297, in _once
        indicator._once()
      File "/Users/buraktas/.virtualenvs/alga-trader/lib/python3.7/site-packages/backtrader/lineiterator.py", line 297, in _once
        indicator._once()
      File "/Users/buraktas/.virtualenvs/alga-trader/lib/python3.7/site-packages/backtrader/lineiterator.py", line 317, in _once
        self.oncestart(self._minperiod - 1, self._minperiod)
      File "/Users/buraktas/.virtualenvs/alga-trader/lib/python3.7/site-packages/backtrader/lineiterator.py", line 327, in oncestart
        self.once(start, end)
      File "/Users/buraktas/.virtualenvs/alga-trader/lib/python3.7/site-packages/backtrader/indicators/basicops.py", line 364, in once
        dst[i] = math.fsum(src[i - period + 1:i + 1]) / period
    IndexError: array assignment index out of rang
    

    I appreciate for any help. Thanks!



  • I am not too sure what would the problem is here, however, I think it may be linked to your feeding of data. I would suggest taking a look at the values, see where the error might be.


  • administrators

    @robin-dhillon said in Getting IndexError: array assignment index out of range in cerebro.run():

    I think it may be linked to your feeding of data

    @burak-aktas said in Getting IndexError: array assignment index out of range in cerebro.run():

    0 2019-05-04 14:00:00  159.81  160.55  158.85  160.00   5281.80
    1 2019-05-04 13:00:00  161.93  162.19  158.00  159.81  12089.68
    2 2019-05-04 12:00:00  162.05  162.27  161.27  162.02   3296.24
    

    @burak-aktas said in Getting IndexError: array assignment index out of range in cerebro.run():

        candle_data = candles.find().sort([('time', -1)])
    


  • @robin-dhillon Hey Robin. I limited the data to get 30 entries. and checked if there is any invalid data. I verified that there is no any error prone data that will cause an issue. Format is the same as I shared above.


  • administrators

    Yes and the format goes from the future to the past ... hence INVALID. But that's not the core of it.

    @burak-aktas said in Getting IndexError: array assignment index out of range in cerebro.run():

    The helloworld example for backtrader can be seen below;

    This is not a helloworld example. Why?

    You are not capable of loading 30 lines of data, so why try connecting to a database, creating indicators, executing orders, receiving notifications and ... and ...

    A helloworld example

    import backtrader as bt
    
    class MyStrategy(bt.Strategy):
        def next(self):
            print(','.join(str(x) for x in [self.datetime.datetime(0), self.data.close[0]]))
    
    cerebro = bt.Cerebro()
    # It would be wise to start reading from a simple plain csv file.
    data = bt.MyChosenDataFeed(dataname='mydataname', otherparams=myotherparams)
    cerebro.adddata(data)
    cerebro.run()
    

    @burak-aktas said in Getting IndexError: array assignment index out of range in cerebro.run():

    class PandasData(bt.feed.DataBase):
        '''
        The ``dataname`` parameter inherited from ``feed.DataBase`` is the pandas
        DataFrame
        '''
    
        params = (
            # Possible values for datetime (must always be present)
            #  None : datetime is the "index" in the Pandas Dataframe
            #  -1 : autodetect position or case-wise equal name
            #  >= 0 : numeric index to the colum in the pandas dataframe
            #  string : column name (as index) in the pandas dataframe
            ('datetime', 0),
        ...
        ...
    

    You probably want to use the real PandasData. Instead of blindly copying the first lines of this page of documentation Docs - Pandas DataFeed Example, which simply serve illustrative purposes (and are prefixed with this header: "The important declarations for the Datafeed." and then followed by "The above excerpt from the PandasData class ..."), you may want to scroll down and see how a pandas dataframe is actually loaded in the full code which comes afterwards.

    See the reference for the PandasData: Docs - Data Feed Reference



  • @backtrader Thanks. I guess I started to understand. I made the code much more clear. I basically converting data from mongodb into a csv then reading data from csv file. However, now I am getting a different error which is

    Traceback (most recent call last):
      File "backtesting.py", line 38, in <module>
        cerebro.adddata(data)
      File "/Users/buraktas/.virtualenvs/alga-trader/lib/python3.7/site-packages/backtrader/cerebro.py", line 758, in adddata
        data.setenvironment(self)
    AttributeError: 'OHLCData' object has no attribute 'setenvironment
    

    And this is the code I am running

    from __future__ import (absolute_import, division, print_function, unicode_literals)
    from datetime import datetime
    import backtrader as bt
    
    
    class TestStrategy(bt.Strategy):
        def next(self):
            print(','.join(str(x) for x in [self.datetime.datetime(0), self.data.close[0]]))
    
    
    class OHLCData(bt.feeds.GenericCSV):
        params = (
            ('dtformat', '%Y-%m-%d %H:%M:%S'),
            ('datetime', 0),
            ('time', -1),
            ('open', 1),
            ('high', 2),
            ('low', 3),
            ('close', 4),
            ('volume', 5),
            ('openinterest', -1),
            ('timeframe', bt.TimeFrame.Minutes),
            ('compression', 60)
        )
    
    
    if __name__ == '__main__':
        cerebro = bt.Cerebro()
        cerebro.addstrategy(TestStrategy)
        cerebro.broker.setcommission(commission=0.001)
    
        data = OHLCData(dataname='/Users/buraktas/algotrader/algotrader/backtesting/test_data.csv')
    
        cerebro.adddata(data)
        cerebro.broker.setcash(1000.0)
        print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue())
        cerebro.run()
        print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue())
        cerebro.plot(style='bar')
    

    And finally here is the example csv file I have;

    datetime,open,high,low,close,volume
    2017-07-15 13:00:00,183.33,189.0,181.34,181.54,9797.22
    2017-07-15 14:00:00,181.52,184.0,178.0,180.26,24246.76
    2017-07-15 15:00:00,180.26,180.27,176.35,178.82,22395.28
    2017-07-15 16:00:00,178.82,182.0,177.11,180.52,11845.78
    2017-07-15 17:00:00,180.52,184.0,178.2,182.49,11625.9
    2017-07-15 18:00:00,182.49,183.2,179.05,179.32,6882.71
    2017-07-15 19:00:00,179.32,181.2,179.28,181.2,8096.29
    2017-07-15 20:00:00,181.19,181.21,175.51,175.51,14454.75
    2017-07-15 21:00:00,175.51,178.12,175.05,176.18,10333.21
    2017-07-15 22:00:00,176.17,177.05,173.5,174.48,15422.67
    2017-07-15 23:00:00,174.49,176.0,167.0,167.72,32336.96
    2017-07-16 00:00:00,168.22,172.0,165.24,165.64,27662.9
    2017-07-16 01:00:00,165.67,171.1,164.0,166.86,24991.78
    2017-07-16 02:00:00,166.86,172.3,166.85,169.78,11534.34
    2017-07-16 03:00:00,169.77,171.21,166.41,167.03,8431.53
    2017-07-16 04:00:00,166.87,168.16,165.05,165.54,6999.98
    2017-07-16 05:00:00,165.33,165.47,160.53,160.79,13150.77
    2017-07-16 06:00:00,160.64,164.0,154.42,160.66,34607.19
    2017-07-16 07:00:00,160.97,162.16,155.0,156.79,11609.98
    2017-07-16 08:00:00,156.78,157.55,154.5,154.53,7494.6
    2017-07-16 09:00:00,154.53,154.63,151.17,152.42,16870.68
    2017-07-16 10:00:00,152.45,152.59,145.0,145.12,21192.75
    2017-07-16 11:00:00,145.28,147.54,137.0,137.99,38179.05
    2017-07-16 12:00:00,137.99,141.7,132.54,134.32,37655.74
    2017-07-16 13:00:00,134.26,155.5,130.26,153.0,77604.66
    2017-07-16 14:00:00,152.12,152.57,140.34,146.93,38863.56
    2017-07-16 15:00:00,146.99,154.56,146.99,152.07,31752.34
    2017-07-16 16:00:00,152.07,163.88,151.01,162.49,34226.85
    2017-07-16 17:00:00,162.47,164.55,157.18,159.0,28391.9
    2017-07-16 18:00:00,158.89,158.93,151.0,153.0,22441.2
    
    


  • Hey @backtrader do you have any chance to look at it? Thanks!


  • administrators



  • @backtrader Thanks! I opened another thread since the error I getting was a different one.


Log in to reply
 

});