For code/output blocks: Use ``` (aka backtick or grave accent) in a single line before and after the block. See: http://commonmark.org/help/
What am I doing wrong with btoandav20 and notify_order()?
-
I'm using btoandav20 in order to place (pyramiding) orders on OANDA.
While in backtesting, everything runs smoothly, I experience issues with the notify_order() when I use "live trading" on the practice server of OANDA (when I am using the OANDA broker).I only get back status submitted, but not accepted or completed, even though the orders are filled and positions opened.
Sometimes I get the status accepted and completed, but I cannot see the pattern behind it.
Also when orders are rejected (e.g. for margin reasons) I get the status accepted and canceledBackground: I'm depending on notify_order() to track positions of multiple feeds/instruments. If there is a more elegant solution, I'm open to suggestions. However, Multi-Data Example and other similar articles / community posts didn't work for me so far.
Below's my code if it helps (I removed the args part, so that I don't exceel the character limit of the post here), I'm stuck for some time now. @dasch are there any issues know with regards to btoandav20 and notify_order()?
Thanks in advance!
Josip#!/usr/bin/env python from __future__ import (absolute_import, division, print_function, unicode_literals) import argparse import configparser import csv import datetime import os import git import sys # Backtrader Specific Modules import backtrader as bt import btoandav20 # Own Modules from helpers.forexcommision import forexSpreadCommisionScheme from indicators import MFI as MFIind from reporting import CMDReport, CSVReport from signals import Signal, Candle StoreCls = btoandav20.stores.OandaV20Store DataCls = btoandav20.feeds.OandaV20Data BrokerCls = btoandav20.brokers.OandaV20Broker class Candles(bt.Strategy): def __init__(self): self.sma = 0 self.inds = {} self.totalpnl = 0 self.init_trail_pips = self.p.init_trail_pips / 10000 self.break_even_trail_pips = self.p.break_even_trail_pips / 10000 self.trailing = self.init_trail_pips self.multitrades = self.p.multitrades self.print_log = self.p.print_log self.profit = self.p.profit_pips / 10000 self.trade_size = self.p.trade_size self.keep_size = self.p.keep_size self.direction = self.p.direction self.long_strat = ["Bullish Harami", "Bullish Kicker", "Bullish Engulfing"] self.short_strat = ["Bearish Harami", "Bearish Kicker", "Bearish Engulfing"] self.open_orders = dict() self.bar_executed = None # To keep track of pending orders and commissions self.order = None self.buycomm = None self.sellcomm = None for i, d in enumerate(self.datas): # ====== INDICATORS ====== self.inds[i] = {} if self.p.sma_signals == True: self.inds[i]["short_term_sma"] = bt.indicators.MovingAverageExponential(d.close, period=self.p.st_sma) self.inds[i]["long_term_sma"] = bt.indicators.MovingAverageSimple(d.close, period=self.p.lt_sma) if self.p.rsi_signal == True: self.inds[i]["rsi"] = bt.indicators.RelativeStrengthIndex(d.close, period=self.p.rsi_period, lowerband=self.p.rsi_lower, upperband=self.p.rsi_upper) if self.p.mfi_signal == True: self.inds[i]["mfi"] = MFIind(period=self.p.mfi_period) if self.p.ema_vol_period: self.inds[i]["ema_vol"] = bt.indicators.MovingAverageExponential(d.volume, period=self.p.ema_vol_period) # === TURTLE INDICATORS === self.inds[i]["turtle_fast_long"] = bt.ind.Highest(d.close, period=self.p.turtle_enter_fast) self.inds[i]["turtle_fast_long_close"] = bt.ind.Lowest(d.close, period=self.p.turtle_exit_fast) self.inds[i]["turtle_fast_short"] = bt.ind.Lowest(d.close, period=self.p.turtle_enter_fast) self.inds[i]["turtle_fast_short_close"] = bt.ind.Highest(d.close, period=self.p.turtle_exit_fast) self.inds[i]["turtle_slow_long"] = bt.ind.Highest(d.close, period=self.p.turtle_enter_slow) self.inds[i]["turtle_slow_long_close"] = bt.ind.Lowest(d.close, period=self.p.turtle_exit_slow) self.inds[i]["turtle_slow_short"] = bt.ind.Lowest(d.close, period=self.p.turtle_enter_slow) self.inds[i]["turtle_slow_short_close"] = bt.ind.Highest(d.close, period=self.p.turtle_exit_slow) self.instrument = self.data._name def log(self, txt, dt=None, dn=None): dn = self.instrument dt = dt or self.datas[0].datetime.datetime(0) if self.p.print_log == True: print('%s %s: %s' % (dt, dn, txt)) def notify_order(self, order): if order.info.ref: logref = f"{order.info.ref}.{order.ref}" else: logref = order.ref self.log('Order ref: {} / Type {} / Status {}'.format( logref, 'Buy' * order.isbuy() or 'Sell', order.getstatusname() )) self.instrument = order.info.instrument if order.status in [order.Submitted, order.Accepted]: if order.info.type in ["long", "short"]: self.open_orders[order.ref]["status"] = order.status self.open_orders[order.ref]["ref"] = order.ref return if order.status in [order.Completed]: if order.info.type in ["long", "short"]: self.open_orders[order.ref]["instrument"] = order.info.instrument self.open_orders[order.ref]["price"] = order.executed.price self.open_orders[order.ref]["units"] = abs(order.executed.size) self.open_orders[order.ref]["cur_size"] = abs(order.executed.size) self.open_orders[order.ref]["status"] = order.status if order.info.type == "long": buyprice = order.executed.price self.open_orders[order.ref]["stop_loss"] = buyprice * (1 - self.init_trail_pips) self.open_orders[order.ref]["take_profit"] = buyprice * (1 + self.profit) self.buycomm = order.executed.comm # CSVReport.write_closed_orders_report() self.log( 'ID: {} - BUY EXECUTED, Price: {}, Cost: {}, Comm {}, SL {}, TP {}, Pos {}'.format( logref, round(order.executed.price, 2), order.executed.value, order.executed.comm, round(self.open_orders[order.ref]["stop_loss"], 2), round(self.open_orders[order.ref]["take_profit"], 2), order.executed.size, ), dn=self.open_orders[order.ref]["instrument"] ) elif order.info.type == "short": sellprice = order.executed.price self.open_orders[order.ref]["stop_loss"] = sellprice * (1 + self.init_trail_pips) self.open_orders[order.ref]["take_profit"] = sellprice * (1 - self.profit) self.sellcomm = order.executed.comm # CSVReport.write_closed_orders_report() self.log( 'ID: {} - SELL EXECUTED, Price: {}, Cost: {}, Comm {}, SL {}, TP {}, Pos {}'.format( logref, round(order.executed.price, 2), order.executed.value, order.executed.comm, round(self.open_orders[order.ref]["stop_loss"], 2), round(self.open_orders[order.ref]["take_profit"], 2), order.executed.size, ), dn=self.open_orders[order.ref]["instrument"] ) elif order.info.type == "close": # CSVReport.write_closed_orders_report() self.log( ' ID: {} - POSITIONS CLOSED, Price: {}, Cost: {}, Comm {}, Net {}, Perc {}, Pos {}'.format( logref, round(order.executed.price,2), order.executed.value, order.executed.comm, round(order.executed.pnl,2), round((order.executed.pnl/order.executed.price*100),2), order.executed.size, ) ) del self.open_orders[order.info.ref] self.log(order.info.desc) elif order.info.type == "trail": self.log( ' ID: {} - POSITIONS CLOSED, Price: {}, Cost: {}, Comm {}, Net {}, Perc {}, Pos {}'.format( logref, round(order.executed.price,2), order.executed.value, order.executed.comm, round(order.executed.pnl,2), round((order.executed.pnl/order.executed.price*100),2), order.executed.size, ) ) self.log(order.info.desc) # CSVReport.write_closed_orders_report() del self.open_orders[order.info.ref] elif order.info.type == "reduce": if self.open_orders[order.info.ref]["cur_size"] == 0: del self.open_orders[order.info.ref] # CSVReport.write_closed_orders_report() self.log('ID: {} - POSITION REDUCED, Price: {}, Cost: {}, Comm {}, Net {}, Perc {}, Pos {}'.format( logref, round(order.executed.price,2), order.executed.value, order.executed.comm, round(order.executed.pnl,2), round((order.executed.pnl/order.executed.price*100),2), order.executed.size, ) ) self.log(order.info.desc) self.bar_executed = len(self) self.order = None elif order.status in [order.Canceled, order.Margin, order.Rejected]: self.log('Order Canceled/Margin/Rejected') del self.open_orders[order.ref] self.order = None def notify_trade(self, trade): if not trade.isclosed: return self.totalpnl += trade.pnl self.log('OPERATION PROFIT, GROSS %.2f, NET %.2f' % (trade.pnl, trade.pnlcomm)) self.log("-----------------------------------------------") self.log('TOTAL P/L, GROSS %.2f,' % (self.totalpnl)) self.log("===============================================") # CMDReport.generate_trades_table() # CSVReport.write_trades_report() def next(self): # This is turned off. If it's on, it stops looping. # if self.order: # return def buy_short(d): # Defines the trigger(s) that lead to that action buytriggers_list = [i for i, val in enumerate((self.bearish_harami_calc, self.bearish_kicker_calc, self.bearish_engulfing_calc,)) if val] buytrigger = [] for tr in buytriggers_list: buytrigger.append(self.short_strat[tr]) buytrigger = str(buytrigger).strip("[]") # In case without brackets self.order = self.sell( exectype=bt.Order.Market, data=d, size=self.trade_size, type='short', instrument=d._name, buytrigger=buytrigger ) # In case with brackets # price = d.close[0] # self.order = self.sell_bracket( # exectype=bt.Order.Market, # stopexec=bt.Order.StopTrail, # stopargs={"trailpercent": self.trailing}, # limitexec=None, # type='short', # instrument=d._name, # buytrigger=buytrigger, # data=d, # size=self.trade_size, # price=price # ) self.open_orders[self.order.ref] = {} self.open_orders[self.order.ref]["units"] = self.order.size self.open_orders[self.order.ref]["direction"] = self.order.info.type self.open_orders[self.order.ref]["instrument"] = d._name self.open_orders[self.order.ref]["buytrigger"] = buytrigger self.open_orders[self.order.ref]["status"] = 0 def buy_long(d): # Defines the trigger(s) that lead to that action buytriggers_list = [i for i, val in enumerate((self.bullish_harami_calc, self.bullish_kicker_calc, self.bullish_engulfing_calc,)) if val] buytrigger = [] for tr in buytriggers_list: buytrigger.append(self.long_strat[tr]) buytrigger = str(buytrigger).strip("[]") # In case without brackets self.order = self.buy( exectype=bt.Order.Market, data=d, size=self.trade_size, type='long', instrument=d._name, buytrigger=buytrigger ) # In case with brackets # price = d.close[0] # self.order = self.buy_bracket( # exectype=bt.Order.Market, # stopexec=bt.Order.StopTrail, # stopargs={"trailpercent": self.trailing}, # limitexec=None, # type='long', # instrument=d._name, # buytrigger=buytrigger, # data=d, # size=self.trade_size, # price=price # ) self.open_orders[self.order.ref] = {} self.open_orders[self.order.ref]["units"] = self.order.size self.open_orders[self.order.ref]["direction"] = self.order.info.type self.open_orders[self.order.ref]["instrument"] = d._name self.open_orders[self.order.ref]["buytrigger"] = buytrigger self.open_orders[self.order.ref]["status"] = 0 def short_trail_m(d, ref, oo): if d.close[0] <= oo["price"] * (1 - self.break_even_trail_pips): self.trailing = self.break_even_trail_pips # Only relevant when buy short with brackets: # self.sell(exectype=bt.Order.StopTrail, trailpercent=self.trailing, replace=oo["ref"]) if d.close[0] >= oo["stop_loss"]: self.order = self.close( data=d, size=oo["cur_size"], instrument=d._name, selltrigger="trailed SL", type='trail', desc=" Short Trailing Stop Reached: {}".format(round(oo["stop_loss"],2)), ref=ref ) oo["cur_size"] = 0 elif d.close[0] * (1 + self.trailing) <= oo["stop_loss"]: oo["stop_loss"] = d.close[0] * (1 + self.trailing) def long_trail_m(d, ref, oo): if d.close[0] >= oo["price"] * (1 + self.break_even_trail_pips): self.trailing = self.break_even_trail_pips # Only relevant when buy long with brackets: # self.sell(exectype=bt.Order.StopTrail, trailpercent=self.trailing, replace=oo["ref"]) if d.close[0] <= oo["stop_loss"]: self.order = self.close( data=d, size=oo["cur_size"], instrument=d._name, selltrigger="trailed SL", type='trail', desc=" Long Trailing Stop Reached: {}".format(round(oo["stop_loss"],2)), ref=ref ) oo["cur_size"] = 0 elif d.close[0] * (1 - self.trailing) >= oo["stop_loss"]: oo["stop_loss"] = d.close[0] * (1 - self.trailing) def short_limit_m(d, ref, oo): if d.close[0] <= oo["take_profit"] and oo["cur_size"] > self.keep_size: self.order = self.close( data=d, size=1, instrument=d._name, selltrigger="TP limit", type='reduce', desc=" Short Limit Reached: {}".format(round(oo["take_profit"],2)), ref=ref ) oo["cur_size"] -= 1 if oo["cur_size"] != 0: # Set a new limit, which includes the stepwise selling of contracts. oo["take_profit"] = oo["take_profit"] * (1 - self.profit) def long_limit_m(d, ref, oo): if d.close[0] >= oo["take_profit"] and oo["cur_size"] > self.keep_size: self.order = self.close( data=d, size=1, selltrigger="TP limit", instrument=d._name, type='reduce', desc=" Long Limit Reached: {}".format(round(oo["take_profit"],2)), ref=ref ) oo["cur_size"] -= 1 if oo["cur_size"] != 0: # Set a new limit, which includes the stepwise selling of contracts. oo["take_profit"] = oo["take_profit"] * (1 + self.profit) def loop_open_orders(d): for ref, oo in self.open_orders.items(): for k, i_name in enumerate(self.datas): if i_name._name == oo["instrument"]: d_pos = k if oo["status"] == 4: # CMDReport.generate_open_positions_table(ref, oo) # CSVReport.write_open_positions_report(ref, oo) if oo["direction"] == "long" and oo["cur_size"] > 0 and oo["take_profit"] and d._name == oo["instrument"]: long_limit_m(d, ref, oo) if oo["direction"] == "long" and oo["cur_size"] > 0 and oo["stop_loss"] and d._name == oo["instrument"]: long_trail_m(d, ref, oo) if oo["direction"] == "short" and oo["cur_size"] > 0 and oo["take_profit"] and d._name == oo["instrument"]: short_limit_m(d, ref, oo) if oo["direction"] == "short" and oo["cur_size"] > 0 and oo["stop_loss"] and d._name == oo["instrument"]: short_trail_m(d, ref, oo) def loop_datas(): for i, d in enumerate(self.datas): self.instrument = d._name long_signal_sma_calc, short_signal_sma_calc = Signal(self.p, self.inds).calc_sma_signals(i) self.short_term_sma = self.inds[i]["short_term_sma"] self.long_term_sma = self.inds[i]["long_term_sma"] long_signal_rsi_calc, short_signal_rsi_calc = Signal(self.p, self.inds).calc_rsi_signal(i) long_signal_mfi_calc, short_signal_mfi_calc = Signal(self.p, self.inds).calc_mfi_signal(i) volume_signal_calc = Signal(self.p, self.inds).calc_ema_vol(i, d) turtle_long_enter_signals_calc, turtle_short_enter_signals_calc, enterLf, enterSf, enterLs, enterSs = Signal(self.p, self.inds).calc_turtle_signals(d, i) self.bearish_harami_calc = Candle().bearish_harami_calc(d) self.bullish_harami_calc = Candle().bullish_harami_calc(d) self.bearish_engulfing_calc = Candle().bearish_engulfing_calc(d) self.bullish_engulfing_calc = Candle().bullish_engulfing_calc(d) self.bearish_kicker_calc = Candle().bearish_kicker_calc(d) self.bullish_kicker_calc = Candle().bullish_kicker_calc(d) d.pos_size = self.getposition(d).size d.pos_price = self.getposition(d).price dn = d._name # === Long / Short SIGNALS === long_signals_calc = (long_signal_sma_calc or volume_signal_calc) and turtle_long_enter_signals_calc short_signals_calc = (short_signal_sma_calc or volume_signal_calc) and turtle_short_enter_signals_calc # Simply log the closing price of the series from the reference self.log('Close, %.2f' % d.close[0]) # Check if an order is pending ... if yes, we cannot send a 2nd one # if self.order: # return candles_log = "\x1b[0;37m-\x1b[0m" # Check if we are in the market if d.pos_size == 0 or self.multitrades: candles_log, candles_reptxt, close_log = calc_logs(d) if any((self.bullish_harami_calc, self.bullish_kicker_calc, self.bullish_engulfing_calc,)) and self.direction == "long" and long_signals_calc: buy_long(d) elif any((self.bearish_harami_calc, self.bearish_kicker_calc, self.bearish_engulfing_calc,)) and self.direction == "short" and short_signals_calc: buy_short(d) loop_open_orders(d) # CMDReport.generate_trades_table(d) # CMDReport.print_ping() loop_datas() # CMDReport.generate_live_table_titles() if not self.bar_executed: return # if d.pos_size != 0 and len(self) >= (self.bar_executed + self.p.number_of_bars): # self.order = self.close(d) # self.order.addinfo(type='close', desc=" Bar Lenght Reached") # return # def stop(self): # self.log('(MA Period %2d) Ending Value %.2f' % # (self.params.maperiod, self.broker.getvalue()), doprint=True) def runstrategy(): args = parse_args() if args.config: parse_config(args) args = parse_args(namespace=args) for arg in vars(args): print(arg, getattr(args, arg)) if args.broker: run_input = input("Continue with these parameters? ") if run_input in ["yes", "y"]: pass else: print("Aborted, not continuing.") return os.system('clear') # Create a cerebro cerebro = bt.Cerebro(tradehistory=True) # storekwargs = dict( # token=args.token, # account=args.account, # practice=not args.live # ) token = os.environ.get("TOKEN") storekwargs = dict( token=token, account=args.account, practice=not args.live ) if not args.no_store: store = StoreCls(**storekwargs) if args.broker: if args.no_store: broker = BrokerCls(**storekwargs) else: broker = store.getbroker() cerebro.setbroker(broker) startcash = broker.getcash() else: startcash = 10000 # Add forex commission scheme comminfo = forexSpreadCommisionScheme(spread=5.0, stocklike=False, acc_counter_currency=True, margin=600.0, mult=50.0) comminfo = forexSpreadCommisionScheme(spread=5.0, stocklike=False, acc_counter_currency=True, margin=600.0) # comminfo = forexSpreadCommisionScheme(spread=5.0, stocklike=False, acc_counter_currency=True, margin=1200.0, mult=50.0) # cerebro.broker.setcommission(commission=0.00002) cerebro.broker.addcommissioninfo(comminfo) timeframe = bt.TimeFrame.TFrame(args.timeframe) # Manage data1 parameters tf1 = args.timeframe1 tf1 = bt.TimeFrame.TFrame(tf1) if tf1 is not None else timeframe cp1 = args.compression1 cp1 = cp1 if cp1 is not None else args.compression if args.resample or args.replay: datatf = datatf1 = bt.TimeFrame.Ticks datacomp = datacomp1 = 1 else: datatf = timeframe datacomp = args.compression datatf1 = tf1 datacomp1 = cp1 fromdate = None if args.fromdate: dtformat = '%Y-%m-%d' + ('T%H:%M:%S' * ('T' in args.fromdate)) fromdate = datetime.datetime.strptime(args.fromdate, dtformat) todate = None if args.todate: dtformat = '%Y-%m-%d' + ('T%H:%M:%S' * ('T' in args.todate)) todate = datetime.datetime.strptime(args.todate, dtformat) duration = None if args.duration: todate = fromdate + datetime.timedelta(days=int(args.duration)) sma_signals = False if args.st_sma and args.lt_sma: sma_signals = True rsi_signal = False if args.rsi_period: rsi_signal = True mfi_signal = False if args.mfi_period: mfi_signal = True turtles = False if args.turtles == True: turtles = True DataFactory = DataCls if args.no_store else store.getdata datakwargs = dict( timeframe=datatf, compression=datacomp, qcheck=args.qcheck, historical=args.historical, fromdate=fromdate, todate=todate, duration=duration, bidask=args.bidask, useask=args.useask, backfill_start=not args.no_backfill_start, backfill=not args.no_backfill, tz=args.timezone ) if args.no_store and not args.broker: # neither store nor broker datakwargs.update(storekwargs) # pass the store args over the data data0 = DataFactory(dataname=args.data0, **datakwargs) data1 = None if args.data1 is not None: if args.data1 != args.data0: datakwargs['timeframe'] = datatf1 datakwargs['compression'] = datacomp1 data1 = DataFactory(dataname=args.data1, **datakwargs) else: data1 = data0 rekwargs = dict( timeframe=timeframe, compression=args.compression, bar2edge=not args.no_bar2edge, adjbartime=not args.no_adjbartime, rightedge=not args.no_rightedge, takelate=not args.no_takelate, ) if args.replay: cerebro.replaydata(data0, **rekwargs) if data1 is not None: rekwargs['timeframe'] = tf1 rekwargs['compression'] = cp1 cerebro.replaydata(data1, **rekwargs) elif args.resample: cerebro.resampledata(data0, **rekwargs) if data1 is not None: rekwargs['timeframe'] = tf1 rekwargs['compression'] = cp1 cerebro.resampledata(data1, **rekwargs) else: cerebro.adddata(data0) if data1 is not None: cerebro.adddata(data1) if args.valid is None: valid = None else: valid = datetime.timedelta(seconds=args.valid) if args.direction == "short": fl_name = "reports/" + str(args.fromdate) + "-" + str(args.todate) + " S" + " " + str(datetime.datetime.now().timestamp()) else: fl_name = "reports/" + str(args.fromdate) + "-" + str(args.todate) + " L" + " " + str(datetime.datetime.now().timestamp()) csv_folder = fl_name strat = cerebro.addstrategy(Candles) try: cerebro.addwriter(bt.WriterFile, csv=True, out=fl_name) except Exception as e: print(e) cerebro.addanalyzer(bt.analyzers.TradeAnalyzer, _name='traa') # Live data ... avoid long data accumulation by switching to "exactbars" thestrats = cerebro.run(exactbars=args.exactbars) if args.exactbars < 1: # plotting is possible if args.plot: pkwargs = dict(style='line') if args.plot is not True: # evals to True but is not True npkwargs = eval('dict(' + args.plot + ')') # args were passed pkwargs.update(npkwargs) cerebro.plot(**pkwargs) thestrat = thestrats[0] portvalue = cerebro.broker.getvalue() pnl = portvalue - startcash if pnl > 0: pnl_str = "+" + str(round(pnl)) else: pnl_str = str(round(pnl)) if args.direction == "short": now_string = "reports/" + str(args.fromdate) + "-" + str(args.todate) + " S" + ": " + pnl_str + " " + str(datetime.datetime.now().timestamp()) else: now_string = "reports/" + str(args.fromdate) + "-" + str(args.todate) + " L" + ": " + pnl_str + " " + str(datetime.datetime.now().timestamp()) os.rename(fl_name, now_string) def parse_config(args): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='Test Oanda v20 integration') # If using Config File config = configparser.ConfigParser(allow_no_value=True) config.read('config/'+args.config) config_sections = config.sections() for cs in config_sections: cfi = config.items(cs) for ia in cfi: ia = list(ia) ia[0] = "--" + ia[0] if ia[1] == "": ia.pop(1) ia = tuple(ia) parser.add_argument(ia[0], action="store_true") else: ia = tuple(ia) if ia[0].strip("--") in INT_ARGS: parser.add_argument(ia[0], type=int, action="store") else: parser.add_argument(ia[0], action="store") item_arg = parser.parse_args(ia, namespace=args) return args if __name__ == '__main__': runstrategy() -
you could log store notifications to see what is going on.
see: https://community.backtrader.com/topic/2967/order-synchronization/12
-
Thank you @dasch, it's well appreciated. I'm still investigating the issue and will get back with my findings.
-
Don't hesitiate to reach out if there is an actual bug, which may be possible. Also if you like, open a issue on github.
