For code/output blocks: Use ``` (aka backtick or grave accent) in a single line before and after the block. See: http://commonmark.org/help/

Error for Looping through Pandas Dataframe



  • All,

    I am trying to follow Curtis Miller's Backtrader example (https://ntguardian.wordpress.com/2017/06/12/getting-started-with-backtrader/). To use more realistic data, I am using the Pandas Data Reader to parse a Tiingo file.

    To add data to cerebro, I am looping through dataframes, cleaning them, parsing them with Pandas Dataframe Reader, and adding them to cerebro. The strategy is designed to trade each instrument. For some reason, my code is only able to run the most recently added data.

    Can anyone help me understand what is wrong with my code?

    import datetime
    from pandas import Series, DataFrame
    import random
    from copy import deepcopy
    from __future__ import (absolute_import, division, print_function,
                            unicode_literals)
    import backtrader.indicators as btind
    import os
    import pandas as pd
    pd.core.common.is_list_like = pd.api.types.is_list_like
    import pandas_datareader.data as pdr
    import matplotlib as plt
    import numpy as np
    import backtrader as bt
    
    TIINGO_API_KEY = 'REDACTED'
    
    class PandasData(bt.feed.DataBase):
        '''
        The ``dataname`` parameter inherited from ``feed.DataBase`` is the pandas
        DataFrame
        '''
    
        params = (
            # Possible values for datetime (must always be present)
            #  None : datetime is the "index" in the Pandas Dataframe
            #  -1 : autodetect position or case-wise equal name
            #  >= 0 : numeric index to the colum in the pandas dataframe
            #  string : column name (as index) in the pandas dataframe
            ('datetime', -1),
    
            # Possible values below:
            #  None : column not present
            #  -1 : autodetect position or case-wise equal name
            #  >= 0 : numeric index to the colum in the pandas dataframe
            #  string : column name (as index) in the pandas dataframe
            ('open', -1),
            ('high', -1),
            ('low', -1),
            ('close', -1),
            ('volume', -1),
            ('openinterest', None),
        )
    
    class SMAC(bt.Strategy):
        """A simple moving average crossover strategy; crossing of a fast and slow moving average generates buy/sell
           signals"""
        params = {"fast": 20, "slow": 50,                  # The windows for both fast and slow moving averages
                  "optim": False, "optim_fs": (20, 50)}    # Used for optimization; equivalent of fast and slow, but a tuple
                                                           # The first number in the tuple is the fast MA's window, the
                                                           # second the slow MA's window
     
        def __init__(self):
            """Initialize the strategy"""
     
            self.fastma = dict()
            self.slowma = dict()
            self.regime = dict()
     
            self._addobserver(True, bt.observers.BuySell)    # CAUTION: Abuse of the method, I will change this in future code (see: https://community.backtrader.com/topic/473/plotting-just-the-account-s-value/4)
     
            if self.params.optim:    # Use a tuple during optimization
                self.params.fast, self.params.slow = self.params.optim_fs    # fast and slow replaced by tuple's contents
     
            if self.params.fast > self.params.slow:
                raise ValueError(
                    "A SMAC strategy cannot have the fast moving average's window be " + \
                     "greater than the slow moving average window.")
     
            for d in self.getdatanames():
     
                # The moving averages
                self.fastma[d] = btind.SimpleMovingAverage(self.getdatabyname(d),      # The symbol for the moving average
                                                           period=self.params.fast,    # Fast moving average
                                                           plotname="FastMA: " + d)
                self.slowma[d] = btind.SimpleMovingAverage(self.getdatabyname(d),      # The symbol for the moving average
                                                           period=self.params.slow,    # Slow moving average
                                                           plotname="SlowMA: " + d)
     
                # Get the regime
                self.regime[d] = self.fastma[d] - self.slowma[d]    # Positive when bullish
     
        def next(self):
            """Define what will be done in a single step, including creating and closing trades"""
            for d in self.getdatanames():    # Looping through all symbols
                pos = self.getpositionbyname(d).size or 0
                if pos == 0:    # Are we out of the market?
                    # Consider the possibility of entrance
                    # Notice the indexing; [0] always mens the present bar, and [-1] the bar immediately preceding
                    # Thus, the condition below translates to: "If today the regime is bullish (greater than
                    # 0) and yesterday the regime was not bullish"
                    if self.regime[d][0] > 0 and self.regime[d][-1] <= 0:    # A buy signal
                        self.buy(data=self.getdatabyname(d))
     
                else:    # We have an open position
                    if self.regime[d][0] <= 0 and self.regime[d][-1] > 0:    # A sell signal
                        self.sell(data=self.getdatabyname(d))
    
    class PropSizer(bt.Sizer):
        """A position sizer that will buy as many stocks as necessary for a certain proportion of the portfolio
           to be committed to the position, while allowing stocks to be bought in batches (say, 100)"""
        params = {"prop": 0.1, "batch": 100}
     
        def _getsizing(self, comminfo, cash, data, isbuy):
            """Returns the proper sizing"""
     
            if isbuy:    # Buying
                target = self.broker.getvalue() * self.params.prop    # Ideal total value of the position
                price = data.close[0]
                shares_ideal = target / price    # How many shares are needed to get target
                batches = int(shares_ideal / self.params.batch)    # How many batches is this trade?
                shares = batches * self.params.batch    # The actual number of shares bought
     
                if shares * price > cash:
                    return 0    # Not enough money for this trade
                else:
                    return shares
     
            else:    # Selling
                return self.broker.getposition(data).size    # Clear the position
    
    cerebro = bt.Cerebro(stdstats=False)    # I don't want the default plot objects
    cerebro.broker.set_cash(1000000)    # Set our starting cash to $1,000,000
    cerebro.broker.setcommission(0.02)
    
    

    This is the problem code (I Think):

    There is something in my adddata call...

    start = datetime.datetime(2010, 1, 1)
    end = datetime.datetime(2016, 10, 31)
    is_first = True
    symbols = ["AAPL", "GOOGL", "MSFT", "AMZN", "YHOO", "SNY", "NTDOY", "IBM", "HPQ", "QCOM", "NVDA"]
    plot_symbols = ["AAPL", "GOOGL", "NVDA"]
    
    
        
    d = {}
    for s in symbols:
        df = pdr.get_data_tiingo(s, start, end, api_key=TIINGO_API_KEY)
        df = df.loc[s]
        df = df.drop(['high','close','divCash','low','open'], axis=1)
        df.columns = ['close', 'high', 'low', 'open', 'adjVolume', 'splitFactor', 'volume']
        d["data" + str(s)] = bt.feeds.PandasData(dataname=df,timeframe=1,openinterest=None)
        data = d["data" + str(s)]
        cerebro.adddata(data)
        if s in plot_symbols:
            if is_first:
                data_main_plot = data
                is_first = False
            else:
                data.plotinfo.plotmaster = data_main_plot
        else:
            data.plotinfo.plot = False
        print(data)
        # Give the data to cerebro
    print(data)
    
    class AcctValue(bt.Observer):
        alias = ('Value',)
        lines = ('value',)
        
        plotinfo = {"plot": True, "subplot": True}
        
        def next(self):
            self.lines.value[0] = self._owner.broker.getvalue()   # Get today's account value (cash + stocks)
    
    cerebro.addobserver(AcctValue)
    cerebro.addstrategy(SMAC)
    cerebro.addsizer(PropSizer)
    cerebro.broker.getvalue()
    
    
    cerebro.run()
    cerebro.plot(iplot=True, volume=False)
    


  • Solution found

    cerebro.adddata(data)
    

    Became:

    
    cerebro.adddata(data, name = s)
    

    The data wasn't named before being added to cerebro.


  • administrators

    In summary:

    @caleb-mock said in Error for Looping through Pandas Dataframe:

        def next(self):
            """Define what will be done in a single step, including creating and closing trades"""
            for d in self.getdatanames():    # Looping through all symbols
                pos = self.getpositionbyname(d).size or 0
    

    You were trying to loop over the names but no names were actually being set ......