Constant rebalance portfolio and cheat_on_open

thomaslee

Hi admins, I have been stuck with writing a strategy that rebalances to uniform weight of every assets at each open. The correct implementation will have the portfolio holding an equal amount of cash in each asset at every open.

At the end of day t, I decide the portfolio weights to be held during the next day t+1. If I issue rebalancing orders now, then I run into problems of order not executing simply because I do not know what the next opening price will be and the system will calculate the size based on OHLC of day t.

Is cheat_on_open the right use case for my strategy?
Can someone help understand backtrader execution flow below?

import backtrader as bt
import datetime as datetime

class CRP(bt.Strategy):
    def __init__(self):
        self.num_assets= len(self.datas)
        self.weights = [1/self.num_assets]*self.num_assets
        # To keep track of pending orders and buy price/commission
        self.order = None
        self.buyprice = None
        self.buycomm = None
        
    def nextstart_open(self):
        self.log('Nextstart_Open')
        self.log('OHLC: '+' '.join(['{:.2f} {:.2f} {:.2f} {:.2f}'.format(a.open[0],a.high[0],a.low[0],a.close[0]) for a in self.datas]))
        for a, w in zip(self.datas, self.weights):
            self.order = self.order_target_percent(data=a, target=w)
            self.log('Order {} Target {:.2f}'.format(a._name, w))

    def next(self):
        self.log('Next')
        self.log('OHLC: '+' '.join(['{:.2f} {:.2f} {:.2f} {:.2f}'.format(a.open[0],a.high[0],a.low[0],a.close[0]) for a in self.datas]))
        self.log('Open Value: '+str([self.getposition(a).size*a.open[0] for a in self.datas]))
        self.log('Close Value: '+str([self.getposition(a).size*a.close[0] for a in self.datas]))

    def next_open(self):
        self.log('Next_Open')
        self.log('OHLC: '+' '.join(['{:.2f} {:.2f} {:.2f} {:.2f}'.format(a.open[0],a.high[0],a.low[0],a.close[0]) for a in self.datas]))
        self.log('Open Value: '+str([self.getposition(a).size*a.open[0] for a in self.datas]))
        self.log('Close Value: '+str([self.getposition(a).size*a.close[0] for a in self.datas]))
        for a, w in zip(self.datas, self.weights):
            self.order = self.order_target_percent(data=a, target=w)
            self.log('Order {} Target {:.2f}'.format(a._name, w))
        
        
    def log(self, arg):
        print('{} {}'.format(self.datetime.date(), arg))
        
    def notify_order(self, order):
        if order.status in [order.Submitted, order.Accepted]:
            # Buy/Sell order submitted/accepted to/by broker - Nothing to do
            return

        # Check if an order has been completed
        # Attention: broker could reject order if not enough cash
        if order.status in [order.Completed]:
            if order.isbuy():
                self.log('Buy Executed @ {:.2f}'.format(order.executed.price))
            elif order.issell():
                self.log('Sell Executed @ {:.2f}'.format(order.executed.price))

            self.bar_executed = len(self)

        elif order.status in [order.Canceled, order.Margin, order.Rejected]:
            self.log('Order Canceled/Margin/Rejected')

        # Write down: no pending order
        self.order = None
            
assets = ['VTI', 'EFA']
cerebro = bt.Cerebro(cheat_on_open=True)
cerebro.broker.set_checksubmit(False)
print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue())
for a in assets:
    data = bt.feeds.YahooFinanceData(dataname=a, fromdate=datetime.datetime(2017,1,1), todate=datetime.datetime(2020,7,26))
    cerebro.adddata(data)
cerebro.addstrategy(CRP)
cerebro.run()
print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue())

Starting Portfolio Value: 10000.00
2017-01-03 Next
2017-01-03 OHLC: 108.76 109.13 108.14 108.80 52.25 52.39 52.16 52.37
2017-01-03 Open Value: [0.0, 0.0]
2017-01-03 Close Value: [0.0, 0.0]
2017-01-03 Nextstart_Open
2017-01-03 OHLC: 109.06 109.73 109.01 109.64 52.65 52.99 52.61 52.97
2017-01-03 Order VTI Target 0.50
2017-01-03 Order EFA Target 0.50
2017-01-04 Buy Executed @ 109.06
2017-01-04 Buy Executed @ 52.65
2017-01-04 Next
2017-01-04 OHLC: 109.06 109.73 109.01 109.64 52.65 52.99 52.61 52.97
2017-01-04 Open Value: [4907.7, 4949.099999999999]
2017-01-04 Close Value: [4933.8, 4979.18]
2017-01-04 Next_Open
2017-01-04 OHLC: 109.51 109.64 108.99 109.42 53.10 53.47 53.10 53.42
2017-01-04 Open Value: [4927.95, 4991.400000000001]
2017-01-04 Close Value: [4923.9, 5021.4800000000005]
2017-01-04 Order VTI Target 0.50
2017-01-04 Order EFA Target 0.50
2017-01-05 Next
2017-01-05 OHLC: 109.51 109.64 108.99 109.42 53.10 53.47 53.10 53.42
2017-01-05 Open Value: [4927.95, 4991.400000000001]
2017-01-05 Close Value: [4923.9, 5021.4800000000005]
2017-01-05 Next_Open
2017-01-05 OHLC: 109.57 110.04 109.22 109.77 53.17 53.29 53.11 53.24
2017-01-05 Open Value: [4930.65, 4997.9800000000005]
2017-01-05 Close Value: [4939.65, 5004.56]
2017-01-05 Order VTI Target 0.50
2017-01-05 Order EFA Target 0.50
2017-01-06 Next
2017-01-06 OHLC: 109.57 110.04 109.22 109.77 53.17 53.29 53.11 53.24
2017-01-06 Open Value: [4930.65, 4997.9800000000005]
2017-01-06 Close Value: [4939.65, 5004.56]
2017-01-06 Next_Open
2017-01-06 OHLC: 109.65 109.68 109.31 109.35 53.04 53.18 52.97 53.16
2017-01-06 Open Value: [4934.25, 4985.76]
2017-01-06 Close Value: [4920.75, 4997.04]
2017-01-06 Order VTI Target 0.50
2017-01-06 Order EFA Target 0.50
2017-01-09 Buy Executed @ 109.65
2017-01-09 Next
2017-01-09 OHLC: 109.65 109.68 109.31 109.35 53.04 53.18 52.97 53.16
2017-01-09 Open Value: [5043.900000000001, 4985.76]
2017-01-09 Close Value: [5030.099999999999, 4997.04]
2017-01-09 Next_Open
2017-01-09 OHLC: 109.33 109.89 109.20 109.43 53.12 53.31 53.12 53.17
2017-01-09 Open Value: [5029.18, 4993.28]
2017-01-09 Close Value: [5033.780000000001, 4997.9800000000005]
2017-01-09 Order VTI Target 0.50
2017-01-09 Order EFA Target 0.50
2017-01-10 Next
2017-01-10 OHLC: 109.33 109.89 109.20 109.43 53.12 53.31 53.12 53.17
2017-01-10 Open Value: [5029.18, 4993.28]
2017-01-10 Close Value: [5033.780000000001, 4997.9800000000005]
2017-01-10 Next_Open
2017-01-10 OHLC: 109.47 109.78 109.07 109.78 53.06 53.46 52.99 53.44
2017-01-10 Open Value: [5035.62, 4987.64]
2017-01-10 Close Value: [5049.88, 5023.36]
2017-01-10 Order VTI Target 0.50
2017-01-10 Order EFA Target 0.50
2017-01-11 Next
2017-01-11 OHLC: 109.47 109.78 109.07 109.78 53.06 53.46 52.99 53.44
2017-01-11 Open Value: [5035.62, 4987.64]
2017-01-11 Close Value: [5049.88, 5023.36]
2017-01-11 Next_Open
2017-01-11 OHLC: 109.59 109.59 108.63 109.49 53.52 53.54 53.29 53.53
2017-01-11 Open Value: [5041.14, 5030.88]
2017-01-11 Close Value: [5036.54, 5031.82]
2017-01-11 Order VTI Target 0.50
2017-01-11 Order EFA Target 0.50
2017-01-12 Next
2017-01-12 OHLC: 109.59 109.59 108.63 109.49 53.52 53.54 53.29 53.53
2017-01-12 Open Value: [5041.14, 5030.88]
2017-01-12 Close Value: [5036.54, 5031.82]
2017-01-12 Next_Open
2017-01-12 OHLC: 109.54 109.93 109.53 109.76 53.57 53.73 53.49 53.72
2017-01-12 Open Value: [5038.84, 5035.58]
2017-01-12 Close Value: [5048.96, 5049.68]
2017-01-12 Order VTI Target 0.50
2017-01-12 Order EFA Target 0.50
2017-01-13 Next
2017-01-13 OHLC: 109.54 109.93 109.53 109.76 53.57 53.73 53.49 53.72
2017-01-13 Open Value: [5038.84, 5035.58]
2017-01-13 Close Value: [5048.96, 5049.68]
2017-01-13 Next_Open
2017-01-13 OHLC: 109.42 109.61 109.06 109.27 53.56 53.59 53.44 53.53
2017-01-13 Open Value: [5033.32, 5034.64]
2017-01-13 Close Value: [5026.42, 5031.82]
2017-01-13 Order VTI Target 0.50
2017-01-13 Order EFA Target 0.50
2017-01-17 Next
2017-01-17 OHLC: 109.42 109.61 109.06 109.27 53.56 53.59 53.44 53.53
2017-01-17 Open Value: [5033.32, 5034.64]
2017-01-17 Close Value: [5026.42, 5031.82]
2017-01-17 Next_Open
2017-01-17 OHLC: 109.37 109.55 109.07 109.52 53.36 53.48 53.21 53.37
2017-01-17 Open Value: [5031.02, 5015.84]
2017-01-17 Close Value: [5037.92, 5016.78]
2017-01-17 Order VTI Target 0.50
2017-01-17 Order EFA Target 0.50
2017-01-18 Next
2017-01-18 OHLC: 109.37 109.55 109.07 109.52 53.36 53.48 53.21 53.37
2017-01-18 Open Value: [5031.02, 5015.84]
2017-01-18 Close Value: [5037.92, 5016.78]
2017-01-18 Next_Open
2017-01-18 OHLC: 109.58 109.68 108.84 109.09 53.28 53.35 53.08 53.24
2017-01-18 Open Value: [5040.68, 5008.32]
2017-01-18 Close Value: [5018.14, 5004.56]
2017-01-18 Order VTI Target 0.50
2017-01-18 Order EFA Target 0.50
2017-01-19 Next
2017-01-19 OHLC: 109.58 109.68 108.84 109.09 53.28 53.35 53.08 53.24
2017-01-19 Open Value: [5040.68, 5008.32]
2017-01-19 Close Value: [5018.14, 5004.56]
2017-01-19 Next_Open
2017-01-19 OHLC: 109.40 109.77 109.17 109.47 53.43 53.56 53.35 53.54
2017-01-19 Open Value: [5032.400000000001, 5022.42]
2017-01-19 Close Value: [5035.62, 5032.76]
2017-01-19 Order VTI Target 0.50
2017-01-19 Order EFA Target 0.50
2017-01-20 Next
2017-01-20 OHLC: 109.40 109.77 109.17 109.47 53.43 53.56 53.35 53.54
2017-01-20 Open Value: [5032.400000000001, 5022.42]
2017-01-20 Close Value: [5035.62, 5032.76]
2017-01-20 Next_Open
2017-01-20 OHLC: 109.32 109.52 108.78 109.18 53.47 53.68 53.35 53.68
2017-01-20 Open Value: [5028.719999999999, 5026.18]
2017-01-20 Close Value: [5022.280000000001, 5045.92]
2017-01-20 Order VTI Target 0.50
2017-01-20 Order EFA Target 0.50
2017-01-23 Next
2017-01-23 OHLC: 109.32 109.52 108.78 109.18 53.47 53.68 53.35 53.68
2017-01-23 Open Value: [5028.719999999999, 5026.18]
2017-01-23 Close Value: [5022.280000000001, 5045.92]
2017-01-23 Next_Open
2017-01-23 OHLC: 109.38 110.25 109.31 110.03 53.59 53.84 53.57 53.81
2017-01-23 Open Value: [5031.48, 5037.46]
2017-01-23 Close Value: [5061.38, 5058.14]
2017-01-23 Order VTI Target 0.50
2017-01-23 Order EFA Target 0.50
2017-01-24 Next
2017-01-24 OHLC: 109.38 110.25 109.31 110.03 53.59 53.84 53.57 53.81
2017-01-24 Open Value: [5031.48, 5037.46]
2017-01-24 Close Value: [5061.38, 5058.14]
2017-01-24 Next_Open
2017-01-24 OHLC: 110.51 111.01 110.43 110.98 54.10 54.39 54.09 54.37
2017-01-24 Open Value: [5083.46, 5085.400000000001]
2017-01-24 Close Value: [5105.08, 5110.78]
2017-01-24 Order VTI Target 0.50
2017-01-24 Order EFA Target 0.50
2017-01-25 Next
2017-01-25 OHLC: 110.51 111.01 110.43 110.98 54.10 54.39 54.09 54.37
2017-01-25 Open Value: [5083.46, 5085.400000000001]
2017-01-25 Close Value: [5105.08, 5110.78]
2017-01-25 Next_Open
2017-01-25 OHLC: 111.01 111.07 110.70 110.82 54.27 54.32 54.09 54.16
2017-01-25 Open Value: [5106.46, 5101.38]
2017-01-25 Close Value: [5097.719999999999, 5091.04]
2017-01-25 Order VTI Target 0.50
2017-01-25 Order EFA Target 0.50
2017-01-26 Next
2017-01-26 OHLC: 111.01 111.07 110.70 110.82 54.27 54.32 54.09 54.16
2017-01-26 Open Value: [5106.46, 5101.38]
2017-01-26 Close Value: [5097.719999999999, 5091.04]
2017-01-26 Next_Open
2017-01-26 OHLC: 110.91 110.94 110.51 110.64 54.11 54.14 53.97 54.07
2017-01-26 Open Value: [5101.86, 5086.34]
2017-01-26 Close Value: [5089.44, 5082.58]
2017-01-26 Order VTI Target 0.50
2017-01-26 Order EFA Target 0.50
2017-01-27 Next
2017-01-27 OHLC: 110.91 110.94 110.51 110.64 54.11 54.14 53.97 54.07
2017-01-27 Open Value: [5101.86, 5086.34]
2017-01-27 Close Value: [5089.44, 5082.58]
2017-01-27 Next_Open
2017-01-27 OHLC: 110.24 110.30 109.25 109.89 53.60 53.72 53.43 53.68
2017-01-27 Open Value: [5071.04, 5038.400000000001]
2017-01-27 Close Value: [5054.94, 5045.92]
2017-01-27 Order VTI Target 0.50
2017-01-27 Order EFA Target 0.50
2017-01-30 Order Canceled/Margin/Rejected

2a. Can someone explain what happens on day 1? Why my buy orders not executed at 108.76 but instead at next day 109.26.
2b. Day 2 start with executing buy at 109.26 so it seems that the previous nextstart_open refers to day 2 (correct function as then the orders issued there are executed on the same day 2)
2c. The portfolio is not balanced at open of each day, and towards the end there are even order rejected.

Thank a lot!

ab_trader

Some ideas and anwers:

yes

2a & 2b. last time I've used cheat-on-close it works as expected, so I had current bar OHLC called. Will take a look into my scripts, but no promises about time frame.

2c. No outputs to show this provided, so my guess would be that you have not enough money to re-balance. Say, at day 1 you have 50 shares of stock 1 and 100 shares of stock 2. Next day you need to buy 5 more shares of stock 1 and sell 10 share of stock 2. Order for stock 1 will go first due to cycling thru datas, but it will be no money for this. Order rejected, portfolio is not re-balanced.

ab_trader

Sorry, I meant cheat-on-open.

thomaslee

I sorted the rebalance so that i will sell first then buy later, so i will sell stock 2 in your example first then i buy stock 1 and since I sell to target 50% the value of stock 2 will be half my portfolio and i buy to target 50% so in principle I will still have sufficient money.

Another execution without using the nextstart_open() method:

Starting Portfolio Value: 10000.00
2017-01-03 Next
2017-01-03 OHLC: 108.76 109.13 108.14 108.80 52.25 52.39 52.16 52.37
2017-01-03 Open Value: [0.0, 0.0]
2017-01-03 Close Value: [0.0, 0.0]
2017-01-03 Next_Open
2017-01-03 OHLC: 109.06 109.73 109.01 109.64 52.65 52.99 52.61 52.97
2017-01-03 Open Value: [0.0, 0.0]
2017-01-03 Close Value: [0.0, 0.0]
2017-01-03 Order VTI Target 0.50
2017-01-03 Order EFA Target 0.50
2017-01-04 Buy Executed @ 109.06
2017-01-04 Buy Executed @ 52.65
2017-01-04 Next
2017-01-04 OHLC: 109.06 109.73 109.01 109.64 52.65 52.99 52.61 52.97
2017-01-04 Open Value: [4907.7, 4949.099999999999]
2017-01-04 Close Value: [4933.8, 4979.18]

So the system always call next() first and then next_open(). I can see the behavior familiar to the one described in the official blog post. That is next_open have access to day 2 OHLC

2017-01-03 Next_Open
2017-01-03 OHLC: 109.06 109.73 109.01 109.64 52.65 52.99 52.61 52.97

so when we issue order

2017-01-03 Order VTI Target 0.50
2017-01-03 Order EFA Target 0.50

they are executed before the open of the day

2017-01-04 Buy Executed @ 109.06
2017-01-04 Buy Executed @ 52.65

after that day 2 begins

2017-01-04 Next
2017-01-04 OHLC: 109.06 109.73 109.01 109.64 52.65 52.99 52.61 52.97

Am I correct here?

ab_trader

In the blog post order is issued and executed at the same day:

2005-04-11 Send Buy, fromopen True, close 3080.6
2005-04-11 Buy Executed at price 3088.47

In your case order is issued day before it is executed. This is bt's default behavior, so I don't see how is coo works.

thomaslee

Thats strange indeed. I strip it to bare minimum (ignored the logging):

    def next_open(self):
        self.buy()

and got this

Starting Portfolio Value: 10000.00
2017-01-03 Next
2017-01-03 OHLC: 108.76 109.13 108.14 108.80 52.25 52.39 52.16 52.37
2017-01-03 Open Value: [0.0, 0.0]
2017-01-03 Close Value: [0.0, 0.0]
2017-01-03 Open Weights: [0.0, 0.0]
2017-01-03 Next_Open
2017-01-03 OHLC: 109.06 109.73 109.01 109.64 52.65 52.99 52.61 52.97
2017-01-03 Open Value: [0.0, 0.0]
2017-01-03 Close Value: [0.0, 0.0]
2017-01-04 Buy Executed @ 109.06
2017-01-04 Next
2017-01-04 OHLC: 109.06 109.73 109.01 109.64 52.65 52.99 52.61 52.97
2017-01-04 Open Value: [109.06, 0.0]
2017-01-04 Close Value: [109.64, 0.0]
2017-01-04 Open Weights: [1.0, 0.0]
2017-01-04 Next_Open
2017-01-04 OHLC: 109.51 109.64 108.99 109.42 53.10 53.47 53.10 53.42
2017-01-04 Open Value: [109.51, 0.0]
2017-01-04 Close Value: [109.42, 0.0]
2017-01-05 Buy Executed @ 109.51
2017-01-05 Next
2017-01-05 OHLC: 109.51 109.64 108.99 109.42 53.10 53.47 53.10 53.42
2017-01-05 Open Value: [219.02, 0.0]
2017-01-05 Close Value: [218.84, 0.0]
2017-01-05 Open Weights: [1.0, 0.0]
2017-01-05 Next_Open
2017-01-05 OHLC: 109.57 110.04 109.22 109.77 53.17 53.29 53.11 53.24
2017-01-05 Open Value: [219.14, 0.0]
2017-01-05 Close Value: [219.54, 0.0]

Running from jupyter notebook and python3.7 installed via pyenv.

thomaslee

One thing I have found is that maybe the order_target_percent will not compute the correct size for me because it is based on closed prices so I may have to write my own sizers that uses open prices.

However, still does not explain why next_open() does not get executed on the same day.

ab_trader

I ran my test script and got the following (on the daily bars):

next_open() is called before the next() on the same date

2019-04-10, Next Open 64.36, 64.85, 64.23, 64.77
2019-04-10, Next 64.36, 64.85, 64.23, 64.77
2019-04-11, Next Open 64.54, 64.60, 63.47, 63.74
2019-04-11, Next 64.54, 64.60, 63.47, 63.74
2019-04-12, Next Open 63.91, 64.80, 63.86, 64.05
2019-04-12, Next 63.91, 64.80, 63.86, 64.05

order issued in the next_open() is executed at the open price of the bar:

2019-04-03, Next Open 62.66, 63.14, 62.22, 62.62
2019-04-03, Next 62.66, 63.14, 62.22, 62.62
2019-04-04, Next Open 62.54, 62.93, 62.06, 62.26
2019-04-04, entry buy order issued
2019-04-04, order status Submitted: 62.26
2019-04-04, order status Accepted: 62.26
2019-04-04, BUY EXECUTED, Price: 62.54, Cost: 62.54, Comm 0.00

So cheat-on-open works as expected. Test script is below - buys each 3rd bar, sells each 5th bar.

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import datetime
import backtrader as bt
import backtrader_addons as bta
import pdb
        

class TestStrategy(bt.SignalStrategy):

    def log(self, txt, dt=None):
        dt = dt or self.datas[0].datetime.date(0)
        print('%s, %s' % (dt.isoformat(), txt))

    def notify_order(self, order):

        os = {order.Submitted: 'Submitted', order.Accepted: 'Accepted',
              order.Completed: 'Completed', order.Cancelled: 'Cancelled',
              order.Margin: 'Margin', order.Rejected: 'Rejected'}

        if order.status in [order.Submitted, order.Accepted]:
            self.log('order status %s: %0.2f' % (os[order.status],
                order.created.price))

        if order.status in [order.Completed]:
            if order.isbuy():
                self.log(
                    'BUY EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' %
                    (order.executed.price,
                     order.executed.value,
                     order.executed.comm))

                self.buyprice = order.executed.price
                self.buycomm = order.executed.comm
            else:
                self.log('SELL EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' %
                         (order.executed.price,
                          order.executed.value,
                          order.executed.comm))

            self.bar_executed = len(self)

        elif order.status in [order.Canceled, order.Margin, order.Rejected]:
            self.log('order status %s: %0.2f' % (os[order.status],
                order.executed.price))

        self.order = None

    def notify_trade(self, trade):
        if not trade.isclosed:
            return

        self.log('OPERATION PROFIT, GROSS %.2f, NET %.2f' %
                 (trade.pnl, trade.pnlcomm))

    def __init__(self):
        pass

    def next(self):
        self.log('Next %0.2f, %0.2f, %0.2f, %0.2f' % (
            self.data.open[0], self.data.high[0], self.data.low[0],
            self.data.close[0]))

    def next_open(self):
        self.log('Next Open %0.2f, %0.2f, %0.2f, %0.2f' % (
            self.data.open[0], self.data.high[0], self.data.low[0],
            self.data.close[0]))

        if not self.position:
            if len(self.data) % 3 == 0:
                self.log('entry buy order issued')
                self.buy()
        else:
            if len(self.data) % 5 == 0:
                self.log('exit sell order issued')
                self.sell()

if __name__ == '__main__':

    cerebro = bt.Cerebro(cheat_on_open=True)
    data = # put your data feed here
    cerebro.adddata(data)
    cerebro.addstrategy(TestStrategy)
    cerebro.run()
    cerebro.plot(style='bar')

ab_trader

@ab_trader said in Constant rebalance portfolio and cheat_on_open:

class TestStrategy(bt.SignalStrategy):

This came by mistake, should be class TestStrategy(bt.Strategy):, but nothing changed.

thomaslee

Thanks a lot i will check your script.

thomaslee

@ab_trader Thanks for your code it works perfectly. One mistake i made is to write the logging function incorrectly so thats why the date are not printed right.

For those who want to use cheat_on_open, be careful about the strategy sizers such as order_target_xxxx they are computed based on Close prices (also stuffs such as getvalue(), etc) so its better to write a similar method to those but using open prices for computation or use completely manual sizing computation for self.buy()/sell().