Getting IndexError: array assignment index out of range in cerebro.run()

[Burak Aktas]

Hi everyone,

I am a backtrader newbie and couldn't solve this problem. I am getting json data from mongodb and converting it into Pandas DataFrame and feeding it into cerebro.

Here is the example data I have

             datetime    open    high     low   close    volume
0 2019-05-04 14:00:00  159.81  160.55  158.85  160.00   5281.80
1 2019-05-04 13:00:00  161.93  162.19  158.00  159.81  12089.68
2 2019-05-04 12:00:00  162.05  162.27  161.27  162.02   3296.24
3 2019-05-04 11:00:00  162.25  162.41  161.90  162.01   7119.30
4 2019-05-04 10:00:00  162.55  162.55  161.45  162.26   4599.58

The helloworld example for backtrader can be seen below;

from datetime import datetime
import backtrader as bt
from mongoengine import connect
from pymongo import MongoClient
import pandas as pd

class TestStrategy(bt.Strategy):
    def log(self, txt, dt=None):
        dt = dt or self.datas[0].datetime.date(0)
        print('%s, %s' % (dt.isoformat(), txt))

    def __init__(self):
        self.dataclose = self.datas[0].close
        self.order = None
        self.buyprice = None
        self.buycomm = None

        self.sma = bt.indicators.SimpleMovingAverage(self.datas[0], period=15)
        self.rsi = bt.indicators.RelativeStrengthIndex()

    def notify_order(self, order):
        if order.status in [order.Submitted, order.Accepted]:
            return

        if order.status in [order.Completed]:
            if order.isbuy():
                self.log(
                    'BUY EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' %
                    (order.executed.price,
                     order.executed.value,
                     order.executed.comm))

                self.buyprice = order.executed.price
                self.buycomm = order.executed.comm
            else:  # Sell
                self.log('SELL EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' %
                         (order.executed.price,
                          order.executed.value,
                          order.executed.comm))

            self.bar_executed = len(self)

        elif order.status in [order.Canceled, order.Margin, order.Rejected]:
            self.log('Order Canceled/Margin/Rejected')

        # Write down: no pending order
        self.order = None

    def notify_trade(self, trade):
        if not trade.isclosed:
            return

        self.log('OPERATION PROFIT, GROSS %.2f, NET %.2f' %
                 (trade.pnl, trade.pnlcomm))

    def next(self):
        self.log('Close, %.2f' % self.dataclose[0])
        print('rsi:', self.rsi[0])
        if self.order:
            return

        if not self.position:
            if (self.rsi[0] < 30):
                self.log('BUY CREATE, %.2f' % self.dataclose[0])
                self.order = self.buy(size=500)

        else:
            if (self.rsi[0] > 70):
                self.log('SELL CREATE, %.2f' % self.dataclose[0])
                self.order = self.sell(size=500)


class PandasData(bt.feed.DataBase):
    '''
    The ``dataname`` parameter inherited from ``feed.DataBase`` is the pandas
    DataFrame
    '''

    params = (
        # Possible values for datetime (must always be present)
        #  None : datetime is the "index" in the Pandas Dataframe
        #  -1 : autodetect position or case-wise equal name
        #  >= 0 : numeric index to the colum in the pandas dataframe
        #  string : column name (as index) in the pandas dataframe
        ('datetime', 0),

        # Possible values below:
        #  None : column not present
        #  -1 : autodetect position or case-wise equal name
        #  >= 0 : numeric index to the colum in the pandas dataframe
        #  string : column name (as index) in the pandas dataframe
        ('open', 1),
        ('high', 2),
        ('low', 3),
        ('close', 4),
        ('volume', 5),
        ('openinterest', None),
    )


if __name__ == '__main__':
    cerebro = bt.Cerebro()
    cerebro.addstrategy(TestStrategy)
    cerebro.broker.setcommission(commission=0.001)

    connect('algotrader')
    client = MongoClient()
    db = client.get_database('algotrader')
    candles = db.get_collection('candles')
    candle_data = candles.find().sort([('time', -1)])

    ohlc = []
    for candle in candle_data:
        ohlc.append([datetime.utcfromtimestamp(candle['time']),
                         candle['open'],
                         candle['high'],
                         candle['low'],
                         candle['close'],
                         candle['volume']])

    df = pd.DataFrame(ohlc, columns = ['datetime', 'open', 'high', 'low', 'close', 'volume'])
    data = PandasData(dataname=df,
                      datetime=0,
                      open=1,
                      high=2,
                      low=3,
                      close=4,
                      volume=5)
    cerebro.adddata(data)
    cerebro.broker.setcash(1000.0)
    print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue())
    cerebro.run()
    print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue())
    cerebro.plot()

Finally, here is the error I am getting;

Traceback (most recent call last):
  File "backtesting.py", line 135, in <module>
    cerebro.run()
  File "/Users/buraktas/.virtualenvs/alga-trader/lib/python3.7/site-packages/backtrader/cerebro.py", line 1127, in run
    runstrat = self.runstrategies(iterstrat)
  File "/Users/buraktas/.virtualenvs/alga-trader/lib/python3.7/site-packages/backtrader/cerebro.py", line 1293, in runstrategies
    self._runonce(runstrats)
  File "/Users/buraktas/.virtualenvs/alga-trader/lib/python3.7/site-packages/backtrader/cerebro.py", line 1652, in _runonce
    strat._once()
  File "/Users/buraktas/.virtualenvs/alga-trader/lib/python3.7/site-packages/backtrader/lineiterator.py", line 297, in _once
    indicator._once()
  File "/Users/buraktas/.virtualenvs/alga-trader/lib/python3.7/site-packages/backtrader/lineiterator.py", line 297, in _once
    indicator._once()
  File "/Users/buraktas/.virtualenvs/alga-trader/lib/python3.7/site-packages/backtrader/lineiterator.py", line 317, in _once
    self.oncestart(self._minperiod - 1, self._minperiod)
  File "/Users/buraktas/.virtualenvs/alga-trader/lib/python3.7/site-packages/backtrader/lineiterator.py", line 327, in oncestart
    self.once(start, end)
  File "/Users/buraktas/.virtualenvs/alga-trader/lib/python3.7/site-packages/backtrader/indicators/basicops.py", line 364, in once
    dst[i] = math.fsum(src[i - period + 1:i + 1]) / period
IndexError: array assignment index out of rang

I appreciate for any help. Thanks!

[Robin Dhillon]

I am not too sure what would the problem is here, however, I think it may be linked to your feeding of data. I would suggest taking a look at the values, see where the error might be.

[backtrader]

@robin-dhillon said in Getting IndexError: array assignment index out of range in cerebro.run():

I think it may be linked to your feeding of data

@burak-aktas said in Getting IndexError: array assignment index out of range in cerebro.run():

0 2019-05-04 14:00:00  159.81  160.55  158.85  160.00   5281.80
1 2019-05-04 13:00:00  161.93  162.19  158.00  159.81  12089.68
2 2019-05-04 12:00:00  162.05  162.27  161.27  162.02   3296.24

@burak-aktas said in Getting IndexError: array assignment index out of range in cerebro.run():

    candle_data = candles.find().sort([('time', -1)])

[Burak Aktas]

@robin-dhillon Hey Robin. I limited the data to get 30 entries. and checked if there is any invalid data. I verified that there is no any error prone data that will cause an issue. Format is the same as I shared above.

[backtrader]

Yes and the format goes from the future to the past ... hence INVALID. But that's not the core of it.

@burak-aktas said in Getting IndexError: array assignment index out of range in cerebro.run():

The helloworld example for backtrader can be seen below;

This is not a helloworld example. Why?

You are not capable of loading 30 lines of data, so why try connecting to a database, creating indicators, executing orders, receiving notifications and ... and ...

A helloworld example

import backtrader as bt

class MyStrategy(bt.Strategy):
    def next(self):
        print(','.join(str(x) for x in [self.datetime.datetime(0), self.data.close[0]]))

cerebro = bt.Cerebro()
# It would be wise to start reading from a simple plain csv file.
data = bt.MyChosenDataFeed(dataname='mydataname', otherparams=myotherparams)
cerebro.adddata(data)
cerebro.run()

@burak-aktas said in Getting IndexError: array assignment index out of range in cerebro.run():

class PandasData(bt.feed.DataBase):
    '''
    The ``dataname`` parameter inherited from ``feed.DataBase`` is the pandas
    DataFrame
    '''

    params = (
        # Possible values for datetime (must always be present)
        #  None : datetime is the "index" in the Pandas Dataframe
        #  -1 : autodetect position or case-wise equal name
        #  >= 0 : numeric index to the colum in the pandas dataframe
        #  string : column name (as index) in the pandas dataframe
        ('datetime', 0),
    ...
    ...

You probably want to use the real PandasData. Instead of blindly copying the first lines of this page of documentation Docs - Pandas DataFeed Example, which simply serve illustrative purposes (and are prefixed with this header: "The important declarations for the Datafeed." and then followed by "The above excerpt from the PandasData class ..."), you may want to scroll down and see how a pandas dataframe is actually loaded in the full code which comes afterwards.

See the reference for the PandasData: Docs - Data Feed Reference

[Burak Aktas]

@backtrader Thanks. I guess I started to understand. I made the code much more clear. I basically converting data from mongodb into a csv then reading data from csv file. However, now I am getting a different error which is

Traceback (most recent call last):
  File "backtesting.py", line 38, in <module>
    cerebro.adddata(data)
  File "/Users/buraktas/.virtualenvs/alga-trader/lib/python3.7/site-packages/backtrader/cerebro.py", line 758, in adddata
    data.setenvironment(self)
AttributeError: 'OHLCData' object has no attribute 'setenvironment

And this is the code I am running

from __future__ import (absolute_import, division, print_function, unicode_literals)
from datetime import datetime
import backtrader as bt


class TestStrategy(bt.Strategy):
    def next(self):
        print(','.join(str(x) for x in [self.datetime.datetime(0), self.data.close[0]]))


class OHLCData(bt.feeds.GenericCSV):
    params = (
        ('dtformat', '%Y-%m-%d %H:%M:%S'),
        ('datetime', 0),
        ('time', -1),
        ('open', 1),
        ('high', 2),
        ('low', 3),
        ('close', 4),
        ('volume', 5),
        ('openinterest', -1),
        ('timeframe', bt.TimeFrame.Minutes),
        ('compression', 60)
    )


if __name__ == '__main__':
    cerebro = bt.Cerebro()
    cerebro.addstrategy(TestStrategy)
    cerebro.broker.setcommission(commission=0.001)

    data = OHLCData(dataname='/Users/buraktas/algotrader/algotrader/backtesting/test_data.csv')

    cerebro.adddata(data)
    cerebro.broker.setcash(1000.0)
    print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue())
    cerebro.run()
    print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue())
    cerebro.plot(style='bar')

And finally here is the example csv file I have;

datetime,open,high,low,close,volume
2017-07-15 13:00:00,183.33,189.0,181.34,181.54,9797.22
2017-07-15 14:00:00,181.52,184.0,178.0,180.26,24246.76
2017-07-15 15:00:00,180.26,180.27,176.35,178.82,22395.28
2017-07-15 16:00:00,178.82,182.0,177.11,180.52,11845.78
2017-07-15 17:00:00,180.52,184.0,178.2,182.49,11625.9
2017-07-15 18:00:00,182.49,183.2,179.05,179.32,6882.71
2017-07-15 19:00:00,179.32,181.2,179.28,181.2,8096.29
2017-07-15 20:00:00,181.19,181.21,175.51,175.51,14454.75
2017-07-15 21:00:00,175.51,178.12,175.05,176.18,10333.21
2017-07-15 22:00:00,176.17,177.05,173.5,174.48,15422.67
2017-07-15 23:00:00,174.49,176.0,167.0,167.72,32336.96
2017-07-16 00:00:00,168.22,172.0,165.24,165.64,27662.9
2017-07-16 01:00:00,165.67,171.1,164.0,166.86,24991.78
2017-07-16 02:00:00,166.86,172.3,166.85,169.78,11534.34
2017-07-16 03:00:00,169.77,171.21,166.41,167.03,8431.53
2017-07-16 04:00:00,166.87,168.16,165.05,165.54,6999.98
2017-07-16 05:00:00,165.33,165.47,160.53,160.79,13150.77
2017-07-16 06:00:00,160.64,164.0,154.42,160.66,34607.19
2017-07-16 07:00:00,160.97,162.16,155.0,156.79,11609.98
2017-07-16 08:00:00,156.78,157.55,154.5,154.53,7494.6
2017-07-16 09:00:00,154.53,154.63,151.17,152.42,16870.68
2017-07-16 10:00:00,152.45,152.59,145.0,145.12,21192.75
2017-07-16 11:00:00,145.28,147.54,137.0,137.99,38179.05
2017-07-16 12:00:00,137.99,141.7,132.54,134.32,37655.74
2017-07-16 13:00:00,134.26,155.5,130.26,153.0,77604.66
2017-07-16 14:00:00,152.12,152.57,140.34,146.93,38863.56
2017-07-16 15:00:00,146.99,154.56,146.99,152.07,31752.34
2017-07-16 16:00:00,152.07,163.88,151.01,162.49,34226.85
2017-07-16 17:00:00,162.47,164.55,157.18,159.0,28391.9
2017-07-16 18:00:00,158.89,158.93,151.0,153.0,22441.2

[Burak Aktas]

Hey @backtrader do you have any chance to look at it? Thanks!

[backtrader]

See here: Community - Getting AttributeError: 'GenericCSV' object has no attribute 'setenvironment' error

[Burak Aktas]

@backtrader Thanks! I opened another thread since the error I getting was a different one.