Backtest with custom columns problem.

Ta Tum

Hello,
I have a predicted file with action columns (it is a ticker for buy/sell/hold) . I would like to load this csv and use feeds data via PandasData Class and I found an error at a cerebro.run(). Could you please suggest me.

Thank you.

my dataframe

dataframe.head()
Out[58]: 
             Datetime    Open    High     Low   Close  Signal
0 2011-06-13 11:00:00  705.56  705.76  703.67  704.41     0.0
1 2011-06-13 12:00:00  704.25  704.52  701.51  701.69     1.0
2 2011-06-13 14:00:00  704.69  709.42  704.39  706.12     0.0
3 2011-06-13 15:00:00  705.86  708.25  705.68  707.52     2.0
4 2011-06-13 16:00:00  707.37  709.80  706.28  708.76     0.0

import pandas as pd
import backtrader as bt
from backtrader.feeds import PandasData

# load dataframe
dataframe = pd.read_csv("model/SET50_H1_2011_2017_Model1_Predicted.csv",names = ["Datetime","Open","High","Low","Close","action"])
dataframe['Datetime'] = pd.to_datetime(dataframe['Datetime'],format='%Y-%m-%d %H:%M:%S')  

class PandasData_Signal(PandasData):
    # Add a 'action' line to the inherited ones from the base class
    lines = ('action',)
    
    # add the parameter to the parameters inherited from the base class
    params = (('action', 7),)


data = PandasData_Signal(dataname=dataframe,
                     #dtformat=('%Y-%m-%d %H:%M:%S'),
                     #timeframe=bt.TimeFrame.Minutes,
                     #tmformat=('%H:%M:%S'),
                     datetime=0,
                     open=1,
                     high=2,
                     low=3,
                     close=4,
                     #volume=-1,
                     action=5,
                     #openinterest=-1
                     #fromdate=date(2017,1,1),
                     #todate=date(2017,1,10)
                    )

class MLSignal(bt.SignalStrategy):
    def log(self, txt, dt=None):
        pass

    def __init__(self):
        # Keep a reference to the "close" line in the data[0] dataseries
        self.dataclose = self.datas[0].close
        self.action = self.datas[0].action

    def next(self):
        self.log(' Close, %.2f' % self.dataclose[0])
        
        if self.order:
            return
        
        if self.action[0] == 1.0:
            self.log('BUY CREATE, %.2f' % self.dataclose[0])
            self.order = self.buy()
        
        if self.action[0] == 2.0:
            self.log('SELL CREATE, %.2f' % self.dataclose[0])
            self.order = self.sell()

cerebro = bt.Cerebro()

# Set our desired cash start
cerebro.broker.setcash(1000000.0)
cerebro.adddata(data)
cerebro.addstrategy(MLSignal)
cerebro.run()
print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue())
print('Final value is %.2f times the initial investment'%(cerebro.broker.getvalue()/1000000.0))
cerebro.plot()

Error code

  File "/Users/tatum/anaconda3/lib/python3.6/site-packages/spyder/utils/site/sitecustomize.py", line 710, in runfile
    execfile(filename, namespace)

  File "/Users/tatum/anaconda3/lib/python3.6/site-packages/spyder/utils/site/sitecustomize.py", line 101, in execfile
    exec(compile(f.read(), filename, 'exec'), namespace)

  File "/Volumes/SANDISK/Algo_Trader_and_Robot_ProjectTraining/backtrader_code/test5.py", line 96, in <module>
    cerebro.run()

  File "/Users/tatum/anaconda3/lib/python3.6/site-packages/backtrader/cerebro.py", line 1127, in run
    runstrat = self.runstrategies(iterstrat)

  File "/Users/tatum/anaconda3/lib/python3.6/site-packages/backtrader/cerebro.py", line 1290, in runstrategies
    self._runonce(runstrats)

  File "/Users/tatum/anaconda3/lib/python3.6/site-packages/backtrader/cerebro.py", line 1691, in _runonce
    strat._oncepost(dt0)

  File "/Users/tatum/anaconda3/lib/python3.6/site-packages/backtrader/strategy.py", line 289, in _oncepost
    self.nextstart()  # only called for the 1st value

  File "/Users/tatum/anaconda3/lib/python3.6/site-packages/backtrader/lineiterator.py", line 342, in nextstart
    self.next()

  File "/Users/tatum/anaconda3/lib/python3.6/site-packages/backtrader/strategy.py", line 1579, in _next_catch
    self._next_custom()

  File "/Volumes/SANDISK/Algo_Trader_and_Robot_ProjectTraining/backtrader_code/test5.py", line 78, in next
    if self.order:

  File "/Users/tatum/anaconda3/lib/python3.6/site-packages/backtrader/lineseries.py", line 461, in __getattr__
    return getattr(self.lines, name)

AttributeError: 'Lines_LineSeries_LineIterator_DataAccessor_Strateg' object has no attribute 'order'

ab_trader

@Ta-Tum looks like self.order need to be initialized in the init():

self.order = None

Ta Tum

@ab_trader, it's work. but I found new problem that a new column to feed is nan (all value). Please suggest me.

Thank you

class PandasData_Signal(PandasData):
    # Add a 'action' line to the inherited ones from the base class
    lines = ('action',)
  
    # add the parameter to the parameters inherited from the base class
    params = (('action', 8),)

data = PandasData_Signal(dataname=dataframe,
                     datetime=0,
                     open=1,
                     high=2,
                     low=3,
                     close=4,
                     #volume=-1,
                     action=5
                     #openinterest=-1
                     #fromdate=date(2017,1,1),
                     #todate=date(2017,1,10)
                    )

error

2017-09-25T16:00:00, Close , 1067.41
2017-09-25T16:00:00, action , nan
2017-09-25T17:00:00, Close , 1070.88
2017-09-25T17:00:00, action , nan

Richard O'Regan

@Ta-Tum Hey mate. Please could you provide your file you read from, then I'll run through it and can probably figure it out quite quickly.

I noticed your line:

dataframe = pd.read_csv("model/SET50_H1_2011_2017_Model1_Predicted.csv",names = ["Datetime","Open","High","Low","Close","action"])

Uses 'action' . and yet your dataframe has 'Signal'
dataframe.head()

Out[58]: 
             Datetime    Open    High     Low   Close  Signal
0 2011-06-13 11:00:00  705.56  705.76  703.67  704.41     0.0
1 2011-06-13 12:00:00  704.25  704.52  701.51  701.69     1.0

This may not be relevant, my familiarity with pandas loading csv with backtrader is not high enough that I can diagnose just from source, I'd need to run, look at errors in realtime & play around with it to figure it out.

Send the .csv file (or a shortened version) and should be easy to fix

Ta Tum

@Richard-O-Regan : For lasted version. I changed a column name to "Signal". I get same error.

Thank you

csv file pic and data
0_1510825628583_Screen Shot 2560-11-16 at 4.46.31 PM.png

Datetime	Open	High	Low	Close	Signal

0 2011-06-13 11:00:00 705.56 705.76 703.67 704.41 0
1 2011-06-13 12:00:00 704.25 704.52 701.51 701.69 0
2 2011-06-13 14:00:00 704.69 709.42 704.39 706.12 0
3 2011-06-13 15:00:00 705.86 708.25 705.68 707.52 0
4 2011-06-13 16:00:00 707.37 709.8 706.28 708.76 0
5 2011-06-14 09:00:00 712.97 718.68 712.87 715.43 0
6 2011-06-14 10:00:00 715.56 715.97 712.11 712.49 0
7 2011-06-14 11:00:00 712.65 716.43 710.17 715.85 0
8 2011-06-14 12:00:00 715.96 716.51 711.77 713.37 0
9 2011-06-14 14:00:00 710.2 713.07 710.17 712.67 0
10 2011-06-14 15:00:00 712.84 714.79 710.87 714.64 0
11 2011-06-14 16:00:00 714.79 724.12 712.5 724.12 1
12 2011-06-15 09:00:00 724.29 724.33 718.51 719.03 0
13 2011-06-15 10:00:00 719.03 721.46 718.51 720.83 0
14 2011-06-15 11:00:00 720.83 723.87 718.51 723.35 0
15 2011-06-15 12:00:00 723.51 723.77 720.77 720.95 0
16 2011-06-15 14:00:00 718.66 722.06 718.51 720.82 0
17 2011-06-15 15:00:00 720.82 722.82 718.81 722.5 0

and coding

import pandas as pd

    

from backtrader.feeds import PandasData
from backtrader.feeds import GenericCSVData


class PandasData_Signal(PandasData):
    # Add a 'pe' line to the inherited ones from the base class
    lines = ('Signal',)
    
    # add the parameter to the parameters inherited from the base class
    params = (('Signal', 8),)

import backtrader as bt


class MLSignal(bt.Strategy):
    def log(self, txt, dt=None):
        dt = dt or self.datas[0].datetime.datetime(0)
        print('%s,%s' % (dt.isoformat(),txt))
        pass

    def __init__(self):
        # Keep a reference to the "close" line in the data[0] dataseries
        self.dataclose = self.datas[0].close
        self.datasignal = self.datas[0].Signal
        self.order = None

    def next(self):
        self.log(' Close, %.2f' % self.dataclose[0])
        self.log(' Signal, %.2f' % self.datasignal[0])
        
        if self.order:
            return
        
        if self.datasignal[0] == 1.0:
            self.log('BUY CREATE, %.2f' % self.dataclose[0])
            self.order = self.buy()
        
        if self.datasignal[0] == 2.0:
            self.log('SELL CREATE, %.2f' % self.dataclose[0])
            self.order = self.sell()

cerebro = bt.Cerebro()



dataframe.to_csv("/Volumes/SANDISK/Algo_Trader_and_Robot_ProjectTraining/Class 2/Ex6/model/SET50_H1_2011_2017_Model1_Predicted_reformat.csv")
dataframe['Datetime'] = pd.to_datetime(dataframe['Datetime'],format='%Y-%m-%d %H:%M:%S')  

print(dataframe.head())

data = PandasData_Signal(dataname=dataframe,
                     #dtformat = ('%Y-%m-%d %H:%M:%S'),
                     datetime=0,
                     open=1,
                     high=2,
                     low=3,
                     close=4,
                     #volume=5,
                     #openinterest=-1
                     Signal=5
                    )


# Set our desired cash start
cerebro.broker.setcash(1000000.0)
cerebro.adddata(data)
# Set our strategy
cerebro.addstrategy(MLSignal)
cerebro.run()
print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue())
print('Final value is %.2f times the initial investment'%(cerebro.broker.getvalue()/1000000.0))
cerebro.plot()

Paska Houso

Seeing that open, close and the others are not written in the code with capital letters, you may try signal in PandasData_Signal

C. L. Tai

hi @Ta-Tum, the parameters of PandasData are programmed as below:

    ('datetime', 0),
    ('open', 1),
    ('high', 2),
    ('low', 3),
    ('close', 4),
    ('volume', 5),
    ('openinterest', 6),

May be you fit ur dataframe as follow:

df = pd.DataFrame(index = dataframe .index)
df['Open'] = dataframe ['Open'].values
df['High'] = dataframe ['High'].values
df['Low'] = dataframe ['Low'].values
df['Close'] = dataframe ['Close'].values
df['Volume'] = dataframe ['Volume'].values
df['openinterest'] = dataframe ['Signal'].values

Then at class MLSignal:

class MLSignal(bt.SignalStrategy):

def log(self, txt, dt=None):
    pass

def __init__(self):
    # Keep a reference to the "close" line in the data[0] dataseries
    self.dataclose = self.datas[0].close
    self.datasignal = self.datas[0].openinterest

def next(self):
    self.log(' Close, %.2f' % self.dataclose[0])
    self.log(' Signal, %.2f' % self.datasignal[0]

    if self.order:
        return
    
    if self.action[0] == 1.0:
        self.log('BUY CREATE, %.2f' % self.dataclose[0])
        self.order = self.buy()
    
    if self.action[0] == 2.0:
        self.log('SELL CREATE, %.2f' % self.dataclose[0])
        self.order = self.sell()

I think this should pass dataframe data signal into the strategy part.
Hope this helps. I've been looking for this solution for quite long too. ;) Thanks