We have the next() which works with updates on upcoming/next data bars, which works with OHLC or bid/ask.

Is there a way I can add a next_orderbook() (say), to push changes in the order book so that the strategy can operate on it?

@ab_trader
Yes, the issue was because of exactbars=True being set. I turned that off and it worked.

Awesome work there!
I don't need to add anything.

I am currently doing some live bot like trading and therefore I though adding a continuous dump feature would be helpful.
This little modification also allows one to dump the position details in a CSV file

class trade_list(bt.Analyzer):
    params = (
        ("continuous", False),
        ("fname", "position_{0}.csv".format(time.strftime("%Y%m%d-%H%M%S"))),
        ("dump", True)
    )
    
    def __init__(self):
        self.trades = []
        self.cumprofit = 0.0
        
    def get_analysis(self):
        return self.trades
    
    def _write_to_file(self):
        if not self.p.dump:
            return
        df = pd.DataFrame(self.trades)
        df.to_csv(self.p.fname, index=False)

    def notify_trade(self, trade):
        if trade.isclosed:
            brokervalue = self.strategy.broker.getvalue()
            dir = "short"
            if trade.history[0].event.size > 0:
                dir = "long"
            pricein = trade.history[len(trade.history)-1].status.price
            priceout = trade.history[len(trade.history)-1].event.price
            datein = bt.num2date(trade.history[0].status.dt)
            dateout = bt.num2date(trade.history[len(trade.history)-1].status.dt)
            if trade.data._timeframe >= bt.TimeFrame.Days:
                datein = datein.date()
                dateout = dateout.date()

            pcntchange = 100 * priceout / pricein - 100
            pnl = trade.history[len(trade.history)-1].status.pnlcomm
            pnlpcnt = 100 * pnl / brokervalue
            barlen = trade.history[len(trade.history)-1].status.barlen
            pbar = pnl / barlen
            self.cumprofit += pnl

            size = value = 0.0
            for record in trade.history:
                if abs(size) < abs(record.status.size):
                    size = record.status.size
                    value = record.status.value

            highest_in_trade = max(trade.data.high.get(ago=0, size=barlen+1))
            lowest_in_trade = min(trade.data.low.get(ago=0, size=barlen+1))
            hp = 100 * (highest_in_trade - pricein) / pricein
            lp = 100 * (lowest_in_trade - pricein) / pricein
            if dir == "long":
                mfe = hp
                mae = lp
            else:
                mfe = -lp
                mae = -hp

            self.trades.append({"ref": trade.ref, "ticker": trade.data._name, "dir": dir,
                                "datein": datein, "pricein": pricein, "dateout": dateout, "priceout": priceout,
                                "chng%": round(pcntchange, 2), "pnl": pnl, "pnl%": round(pnlpcnt, 2),
                                "size": size, "value": value, "cumpnl": self.cumprofit,
                                "nbars": barlen, "pnl/bar": round(pbar, 2),
                                "mfe%": round(mfe, 2), "mae%": round(mae, 2)})
            if self.p.continuous:
                self._write_to_file()

    def stop(self):
        if not self.p.continuous:
            self._write_to_file()

@totoro It depends on what you are doing inside the loop. If you are making some calculations like std devs, atr etc then I don't think that even with 100's of data feed your lag could reach 10s. If you are running some neural network based complex calculations then it can though.
You can use something like the time command to get an idea about your execution time.

@totoro You would have to process each data feed separately and there is no workaround that in any language or using any framework. Even using pandas you have to filter the DataFrame according to the symbol being processed in this round.

If your strategy relies on microseconds precision then you are entering the realm of HFT and in that case, you have to evaluate if Python is really the correct tool of your choice.

@ab_trader
I am testing this on a live bot.

I ran into the following issue:

[TradeHistory([('status', AutoOrderedDict([('status', 1), ('dt', 736955.651388889), ('barlen', 0), ('size', 46), ('price', 6338.0), ('value', 291548.0), ('pnl', 0.0), ('pnlcomm', 0.0), ('tz', None)])), ('event', AutoOrderedDict([('order', <backtrader.order.BuyOrder object at 0x112DBA70>), ('size', 46), ('price', 6338), ('commission', 0.0)]))]), TradeHistory([('status', AutoOrderedDict([('status', 2), ('dt', 736955.7006944445), ('barlen', 71), ('size', 0), ('price', 6338.0), ('value', 0.0), ('pnl', 506.0), ('pnlcomm', 506.0), ('tz', None)])), ('event', AutoOrderedDict([('order', <backtrader.order.SellOrder object at 0x132FBB30>), ('size', -46), ('price', 6349), ('commission', 0.0)]))])] ref:1
data:<backmex.feed.BitMEXData object at 0x1129C5F0>
tradeid:0
size:0
price:6338.0
value:0.0
commission:0.0
pnl:506.0
pnlcomm:506.0
justopened:False
isopen:False
isclosed:True
baropen:101
dtopen:736955.651388889
barclose:172
dtclose:736955.7006944445
barlen:71
historyon:True
history:[TradeHistory([('status', AutoOrderedDict([('status', 1), ('dt', 736955.651388889), ('barlen', 0), ('size', 46), ('price', 6338.0), ('value', 291548.0), ('pnl', 0.0), ('pnlcomm', 0.0), ('tz', None)])), ('event', AutoOrderedDict([('order', <backtrader.order.BuyOrder object at 0x112DBA70>), ('size', 46), ('price', 6338), ('commission', 0.0)]))]), TradeHistory([('status', AutoOrderedDict([('status', 2), ('dt', 736955.7006944445), ('barlen', 71), ('size', 0), ('price', 6338.0), ('value', 0.0), ('pnl', 506.0), ('pnlcomm', 506.0), ('tz', None)])), ('event', AutoOrderedDict([('order', <backtrader.order.SellOrder object at 0x132FBB30>), ('size', -46), ('price', 6349), ('commission', 0.0)]))])]
status:2

  self._runnext(runstrats)
  File "D:\envs\twenv\lib\site-packages\backtrader\cerebro.py", line 1623, in _runnext
    self._brokernotify()
  File "D:\envs\twenv\lib\site-packages\backtrader\cerebro.py", line 1370, in _brokernotify
    owner._addnotification(order, quicknotify=self.p.quicknotify)
  File "D:\envs\twenv\lib\site-packages\backtrader\strategy.py", line 553, in _addnotification
    self._notify(qorders=qorders, qtrades=qtrades)
  File "D:\envs\twenv\lib\site-packages\backtrader\strategy.py", line 577, in _notify
    analyzer._notify_trade(trade)
  File "D:\envs\twenv\lib\site-packages\backtrader\analyzer.py", line 170, in _notify_trade
    self.notify_trade(trade)
  File "D:\envs\twenv\tradework\lib\backtrader\helpers.py", line 168, in notify_trade
    highest_in_trade = max(trade.data.high.get(ago=0, size=barlen+1))
  File "D:\envs\twenv\lib\site-packages\backtrader\linebuffer.py", line 182, in get
    return list(islice(self.array, start, end))
ValueError: Indices for islice() must be None or an integer: 0 <= x <= sys.maxsize.

The first two objects are the printout of 'trade' and 'trade.history', if it helps in finding the issue.

EDIT: I am using this with exactbars=True option in cerebro and maybe this is causing the error. I will try the same with this option off and let it be known here.

EDIT: Post created by mistake. As I cannot delete it, I have edited it out.

@ab_trader
Thanks for the comments. This is a good use case and I will be thinking of how to add this.

Sometimes during backtesting, we may want to know the timings of entry/exit along with other position details so we can verify it with real data.
This is even more imperative when the strategy is being used to generate signals on T - 1 day and then later confirmed in the market how it would have performed on day T.

I have written a little utility(Analyzer) which outputs all these details in a CSV file. Hope this is useful.

import time
import queue
import pandas as pd
import backtrader as bt
from collections import defaultdict

class PositionAnalyzer(bt.Analyzer):
    params = (
        ("continuous", False),
        ("fname", "position_{0}.csv".format(time.strftime("%Y%m%d-%H%M%S")))
    )

    def __init__(self):
        super(PositionAnalyzer, self).__init__()
        self.order_map = defaultdict(queue.Queue)
        self.df = pd.DataFrame(columns=["instrument", "entry_dt", "exit_dt", "size", "value",
                                        "direction", "entry_price", "exit_price", "pnl", "pnl_comm"])

    def notify_order(self, order):
        if not order.status == order.Completed:
            return
        self.order_map[order.data].put(order)

    def notify_trade(self, trade):
        if not trade.isclosed:
            return
        entry_order = self.order_map[trade.data].get()
        exit_order = self.order_map[trade.data].get()
        instrument = trade.data._dataname
        entry_dt = bt.num2date(entry_order.executed.dt)
        exit_dt = bt.num2date(exit_order.executed.dt)
        size = entry_order.executed.size
        value = entry_order.executed.size * entry_order.executed.price
        direction = "BUY" if entry_order.isbuy() else "SELL"
        entry_price = entry_order.executed.price
        exit_price = exit_order.executed.price
        pnl = round(trade.pnl, 2)
        pnl_comm = round(trade.pnlcomm, 2)
        self.df.loc[len(self.df)] = [instrument, entry_dt, exit_dt, size, value,
                                     direction, entry_price, exit_price, pnl, pnl_comm]
        if self.p.continuous:
            self.df.to_csv(self.p.fname, index=False)

    def stop(self):
        if not self.p.continuous:
            self.df.to_csv(self.p.fname, index=False)

    def get_analysis(self):
        return self.df

Example of a generated output:

instrument,entry_dt,exit_dt,size,value,direction,entry_price,exit_price,pnl,pnl_comm
ADANIPORTS,2018-05-22 05:30:00,2018-05-22 05:30:00,-1,-381.5,SELL,381.5,381.35,0.15,0.15
ADANIPORTS,2018-05-23 05:30:00,2018-05-23 05:30:00,-1,-377.75,SELL,377.75,374.05,3.7,3.7
ADANIPORTS,2018-05-24 05:30:00,2018-05-24 05:30:00,-1,-373.75,SELL,373.75,371.5,2.25,2.25
ADANIPORTS,2018-06-07 05:30:00,2018-06-07 05:30:00,1,378.0,BUY,378.0,381.0,3.0,3.0
ADANIPORTS,2018-06-14 05:30:00,2018-06-14 05:30:00,-1,-381.65,SELL,381.65,374.6,7.05,7.05
ADANIPORTS,2018-06-15 05:30:00,2018-06-15 05:30:00,-1,-371.3,SELL,371.3,371.65,-0.35,-0.35
ADANIPORTS,2018-06-21 05:30:00,2018-06-21 05:30:00,1,369.6,BUY,369.6,366.5,-3.1,-3.1

I am saving reference to open positions in the stocks using a dictionary which maps 'data' to True/False denoting open positions.

At the start of next(), I close all the positions in 'data' which have open positions using the above dict.

    def next(self):
        for d in self.pos_map:
            if self.pos_map[d]:
                self.close(data=d)
                self.pos_map[d] = False
        # more code to create orders using strategy

I hope I am doing it right and it will close the orders at the close which were executed in the last next() cycle(the bar after whose completion this next() was called and in which my main orders were executed)