For code/output blocks: Use ``` (aka backtick or grave accent) in a single line before and after the block. See: http://commonmark.org/help/
Init data gets not recalculated in optstrategy
-
Hello Everyone,
I have an issue with the optstrategy. The code seems to reuse the self.ranks dictionary from the init:
self.ranks = {d: m / v for d, v, m in zip(self.stocks, vs, ms)}The Code:
# <<<Momentum Strategy section>>> class St(bt.Strategy): params = dict( selcperc=0.50, # percentage of stocks to select from the universe rperiod=1, # period for the returns calculation, default 1 period vperiod=11, # lookback period for volatility - default 36 periods mperiod=16, # lookback period for momentum - default 90 periods momentum=Momentum, # parametrize the momentum and its period reserve=globalparams["reserve"], # 5% reserve capital monthdays=[1], monthcarry=True, when=bt.timer.SESSION_START, benchmarkstop=False, # If true, no stocks will be bought and no rebalancing will be done if benchmark is below SMAperiod SMAperiod=200, benchmark_bond=True, # Sell all Stocks and buy Bonds jump_momentum=True, # If true, after a time of jump_one (30 days x jump_one) in every month, all the money will be directed to the best performing stock. Rule for that: # In Excel, this is a 0.6 x month return of fund with best past 3 month return plus 0.4 x return of fund with best return, month to date. jump_one=0.6, printlog=True, ) def __init__(self): self.bench = self.data0 self.bond = self.data1 self.stocks = self.datas[2:] # calculate 1st the amount of stocks that will be selected self.selnum = int(len(self.stocks) * self.p.selcperc) # allocation perc per stock # reserve kept to make sure orders are not rejected due to # margin. Prices are calculated when known (close), but orders can only # be executed next day (opening price). Price can gap upwards self.perctarget = (1.0 - self.p.reserve) / self.selnum # This is the set up of the timer that makes the strategy being executed at the given time self.add_timer( when=self.p.when, monthdays=self.p.monthdays, monthcarry=self.p.monthcarry ) jump = True # returns, volatilities and momentums rs = [bt.ind.PctChange(d, period=self.p.rperiod) for d in self.stocks] vs = [bt.ind.StdDev(ret, period=self.p.vperiod) for ret in rs] #ms = [bt.ind.ROC(d, period=self.p.mperiod) for d in self.datas] ms = [self.p.momentum(d, period=self.p.mperiod) for d in self.stocks] self.bench_sma = bt.ind.SMA(self.data0, period=self.p.SMAperiod) # simple rank formula: (momentum * net payout) / volatility # the highest ranked: low vol, large momentum, large payout self.ranks = {d: m / v for d, v, m in zip(self.stocks, vs, ms)} #TODO: does it perform better without the volatility? self.bench_filter = self.bench < self.bench_sma def prenext(self): # call next() even when data is not available for all tickers self.next() def log(self, arg): if self.p.printlog: print('{} {}'.format(self.datetime.date(), arg)) # This section is for logging of orders in greater detail to figure out whether the strategy is actually having no problem with orders def notify_order(self, order): if order.status in [order.Accepted]: # Buy/Sell order submitted/accepted to/by broker - Nothing to do return if order.status in [order.Submitted]: if order.isbuy(): dt, dn = self.datetime.date(), order.data._name print('Buy {} {} {} Price {:.2f} Value {:.2f} Size {} Cash {:.2f}'.format( order.getstatusname(), dt, dn, order.created.price, order.created.size * order.created.price , order.created.size, self.broker.getcash())) if order.issell(): dt, dn = self.datetime.date(), order.data._name print('Sell {} {} {} Price {:.2f} Value {:.2f} Size {}'.format( order.getstatusname(), dt, dn, order.created.price, order.created.size * order.created.price, order.created.size)) # Buy/Sell order submitted/accepted to/by broker - Nothing to do return # Check if an order has been completed # Attention: broker could reject order if not enough cash if order.status in [order.Completed]: if order.isbuy(): dt, dn = self.datetime.date(), order.data._name print('Buy {} {} Price {:.2f} Value {:.2f} Size {}'.format( dt, dn, order.executed.price, order.executed.value, order.executed.size)) if order.issell():# Sell dt, dn = self.datetime.date(), order.data._name print('Sell {} {} Price {:.2f} Value {:.2f} Size {}'.format( dt, dn, order.executed.price, order.executed.value, order.executed.size)) elif order.status in [order.Canceled, order.Margin, order.Rejected]: self.log('Order Canceled/Margin/Rejected') # This is the function using the timer to execute the rebalance def notify_timer(self, timer, when, *args, **kwargs): #print('strategy notify_timer with tid {}, when {} _getminperstatus {}'. # format(timer.p.tid, when, int(self._getminperstatus()))) if self._getminperstatus() < 0: self.rebalance() def next(self): pass # must be filled with a pass # Actual order giving by a ranking takes place here def rebalance(self): #if jump == True: # Enter Jump Code here # sort data and current rank ranks = sorted( self.ranks.items(), # get the (d, rank), pair key=lambda x: x[1][0], # use rank (elem 1) and current time "0" reverse=True, # highest ranked 1st ... please ) # put top ranked in dict with data as key to test for presence rtop = dict(ranks[:self.selnum]) # For logging purposes of stocks leaving the portfolio rbot = dict(ranks[self.selnum:]) # prepare quick lookup list of stocks currently holding a position posdata = [d for d, pos in self.getpositions().items() if pos] if self.p.benchmarkstop: for d in (d for d in posdata): if "Bond" == d._name and self.bench_filter: return else: if "Bond" == d._name and not self.bench_filter: self.order_target_percent("Bond", target=0.0) self.log('Leave {} due to end of down period'.format(d._name)) return # Triple Momentum: If Benchmark index is below SMA, nothing will be bought or rebalanced if self.p.benchmarkstop: if self.bench_filter: #print('SMA {} - Bench {}'.format(self.bench_sma[0], self.bench[0])) if self.p.benchmark_bond: for d in posdata: self.log('Leave {} due to switch to Bonds'.format(d._name)) self.order_target_percent(d, target=0.0) self.order_target_percent("Bond", target=0.95) self.log('Buy Bond') bond_flag = True return #Code stops here and skips rebalancing und buying # remove those no longer top ranked # do this first to issue sell orders and free cash for d in (d for d in posdata if d not in rtop): self.log('Leave {} - Rank {:.2f}'.format(d._name, rbot[d][0])) self.order_target_percent(d, target=0.0) # rebalance those already top ranked and still there for d in (d for d in posdata if d in rtop): self.log('Rebal {} - Rank {:.2f}'.format(d._name, rtop[d][0])) self.order_target_percent(d, target=self.perctarget) del rtop[d] # remove it, to simplify next iteration # issue a target order for the newly top ranked stocks # do this last, as this will generate buy orders consuming cash for d in rtop: self.log('Enter {} - Rank {:.2f}'.format(d._name, rtop[d][0])) self.order_target_percent(d, target=self.perctarget) def stop(self): pnl = round(self.broker.getvalue() - globalparams["cash"],2) print('Final PnL: {}'.format( pnl)) def run(args=None): #optreturn=False otherwise the heatmap doesn't work cerebro = bt.Cerebro(maxcpus=1, preload=False, optdatas=False, optreturn=False, stdstats=True) # <<<Data loading section>>> # Parse from/to-date fromdate = datetime.datetime(2014, 11, 3) todate = datetime.datetime(2020, 6, 18) # Add SPY/QQQ as "Benchmark" df0 = pd.read_csv(r'C:\Users\MMD\PycharmProjects\Trading\Data Mining\Data\QQQ.csv', index_col=0, parse_dates=True) benchdata = bt.feeds.PandasData(dataname=df0,name="QQQ",fromdate=fromdate, todate=todate, plot=False) cerebro.adddata(benchdata) # Add TMF as "Bond" df1 = pd.read_csv(r'C:\Users\MMD\PycharmProjects\Trading\Data Mining\Data\TMF.csv', index_col=0, parse_dates=True) bonddata = bt.feeds.PandasData(dataname=df1,name="Bond",fromdate=fromdate, todate=todate, plot=False) cerebro.adddata(bonddata) # add all the data files available in the directory datadir for fname in glob.glob(os.path.join(r'C:\Users\MMD\PycharmProjects\Trading\Data Mining\Data\Momentum', '*')): df = pd.read_csv(fname, index_col=0, parse_dates=True) if len(df)>200: cerebro.adddata(bt.feeds.PandasData(dataname=df,name=os.path.basename(fname).replace(".csv", ""),fromdate=fromdate, todate=todate, plot=False)) #print(os.path.basename(fname).replace(".csv", "")) #prints the name of the added csv file # <<<Cerebro loading section>>> # add strategy #cerebro.addstrategy(HoldAllStrategy, buy_date=datetime.date(2013, 3, 31)) #cerebro.addstrategy(HSt, buy_date=datetime.date(2014, 11, 3)) #cerebro.addstrategy(St) cerebro.optstrategy(St, mperiod=range(15, 17), vperiod=range(10, 12)) # set the cash, cheat on close and commission cerebro.broker.setcash(globalparams["cash"]) cerebro.broker.set_coc(True) cerebro.broker.setcommission(commission=globalparams["commission"]) # Adding Analysers #cerebro.addanalyzer(bt.analyzers.SharpeRatio, riskfreerate=0.0, _name='sharpe') cerebro.addanalyzer(btanal.PyFolio) # Needed to use PyFolio cerebro.addanalyzer(btanal.TradeAnalyzer) # Analyzes individual trades cerebro.addanalyzer(bt.analyzers.AnnualReturn, _name='annual_return') # If you want to have all data written into a log file #cerebro.addwriter(bt.WriterFile, csv=True, out='log.csv') cerebro.addobserver(bt.observers.Benchmark, data=benchdata, timeframe=bt.TimeFrame.NoTimeFrame) results = cerebro.run(maxcpus=1)#maxcpu=1 otherwise pickling multiprocessing errors ''' final_results_list = [] for run in results: for strategy in run: PnL = round(strategy.broker.get_value() - globalparams["cash"], 2) #my_dict = strategy.analyzers.sharpe.get_analysis() my_dict = strategy.analyzers.annual_return.get_analysis() annual_returns = [v for _, v in my_dict.items() if v != 0] average_annual_return = sum(annual_returns) / len(annual_returns) final_results_list.append([strategy.params.mperiod, strategy.params.vperiod, PnL, round(average_annual_return*100, 2)]) # Adjust here the Heatmap variables params y-axis and x-axis #2 my_heatmap(final_results_list) ''' # <<<Performance analysing section section>>> #cerebro.plot() # Basic performance evaluation ... final value ... minus starting cash pnl = cerebro.broker.get_value() - globalparams["cash"] print('Profit ... or Loss: {:.2f}'.format(pnl)) # Quantstats thanks to https://algotrading101.com/learn/backtrader-for-backtesting/ # Does not work with optstrategy ''' returns, positions, transactions, gross_lev = results[0].analyzers.pyfolio.get_pf_items() returns.index = returns.index.tz_convert(None) qs.reports.html(returns, output='stats.html', title='Momentum') webbrowser.open('stats.html') ''' # Pyfolio if needed ''' returns, positions, transactions, gross_lev = results[0].analyzers.pyfolio.get_pf_items() benchmark_rets = pd.Series([0.00004] * len(returns.index), index=returns.index) pf.create_full_tear_sheet(returns, positions, transactions, benchmark_rets=benchmark_rets) ''' # <<<Execute starting section>>> if __name__ == '__main__': run()The log (first run is fine... but in the second run the old data starts seeping in):
2020-05-29 Rebal ARKK - Rank 257.37 2020-05-29 Rebal ARKW - Rank 194.29 Sell Submitted 2020-06-02 ARKK Price 64.80 Value -129.60 Size -2 Sell 2020-06-02 ARKK Price 64.80 Value 107.72 Size -2 Final PnL: 9594.99 (end of 1. run) 2020-06-18 Enter ARKW - Rank 81.49 2020-06-18 Enter QQQ - Rank 42.31 2015-08-31 Enter ARKK - Rank 5.77 2015-08-31 Enter ARKW - Rank 3.85 Buy Submitted 2015-09-02 ARKK Price 19.37 Value 4745.65 Size 245 Cash 468.19 Buy Submitted 2015-09-02 ARKW Price 21.33 Value 4735.04 Size 222 Cash 468.19 2015-09-02 Order Canceled/Margin/Rejected 2015-09-02 Order Canceled/Margin/Rejected 2020-06-18 Rebal ARKW - Rank 81.49 2020-06-18 Rebal QQQ - Rank 42.31 2015-09-30 Rebal ARKW - Rank 15.38 2015-09-30 Rebal QQQ - Rank 17.73 Buy Submitted 2015-10-02 ARKW Price 21.13 Value 21.13 Size 1 Cash 425.72 Buy 2015-10-02 ARKW Price 21.13 Value 21.13 Size 1 2020-06-18 Rebal ARKW - Rank 81.49 2020-06-18 Rebal QQQ - Rank 42.31 2015-10-30 Leave ARKW - Rank 105.01 2015-10-30 Rebal QQQ - Rank 364.09 2015-10-30 Enter EUNL.DE - Rank 338.88 Sell Submitted 2015-11-03 ARKW Price 23.03 Value -5158.72 Size -224 Sell Submitted 2015-11-03 QQQ Price 114.61 Value -114.61 Size -1 Buy Submitted 2015-11-03 EUNL.DE Price 38.21 Value 5196.56 Size 136 Cash 517.89 Sell 2015-11-03 ARKW Price 23.03 Value 4780.67 Size -224 Sell 2015-11-03 QQQ Price 114.61 Value 101.05 Size -1 Buy 2015-11-03 EUNL.DE Price 38.21 Value 5196.56 Size 136 2020-06-18 Leave EUNL.DE - Rank 0.01 2020-06-18 Rebal ARKW - Rank 81.49 2020-06-18 Rebal QQQ - Rank 42.31 2015-11-30 Leave ARKW - Rank 90.69 2015-11-30 Leave QQQ - Rank 39.61 2015-11-30 Rebal EUNL.DE - Rank 143.75 2015-11-30 Enter ARKK - Rank 108.25I am not sure how to deal with it...
First I tried to change the cerebro settings:
https://www.backtrader.com/docu/cerebro/Didn't work for me...
I tried to put it in Nextstart(self) ... didn't work
Then I tried to empty or set the dictionaries to none... also didn't work
Is there another way?
Thank you very much in advance!
-
@Jonny8 said in Init data gets not recalculated in optstrategy:
pnl = cerebro.broker.get_value() - globalparams["cash"]
Just a note:
I'm not sure it is a good idea to query the info from the original Cerebro instance's broker.
In multi-CPU case the original Cerebro instance's broker is not updated. In single CPU case (as in the code above), although the same broker instance is used for all the optimization parameter permutation runs, the broker state is reset for each permutation. So querying the broker in this case will return the value for the last optimization run.
It is better to rely only on the data gathered by the analyzers.
