@taylerallen6 What do you mean by depreciated and no longer works ? I had a mod suggest to me the original Oanda API just last week. That said I haven't tried implementation yet. Also, click the link above you, go to the issues section, and under "Which features are missing" is a thread of their progress.

Here is the CSV-based version, however I can't figure out why, but sometimes the entire file isn't flushed. This version works much faster. import backtrader as bt import re import os import os.path from typing import List import datetime as dt import logging import math import pytz import csv def getTableName(data): name = getattr(data,'_name',data.__class__.__name__).strip() # Replace spaces with underscores name = re.sub('\s+','_',name) # Remove invalid characters name = re.sub('[^0-9a-zA-Z_]', '', name) # Remove leading characters until we find a letter or underscore name = re.sub('^[^a-zA-Z_]+', '', name) return name def getTableColumns(data): columns = dict( datetime = dt.datetime ) for line in data.lines.getlinealiases(): if line == 'datetime': continue columns[line] = float return columns class PersistToCSV(bt.Observer): lines = ('ignored',) params = dict( debug = False, rootpath = '/path/to/csv/output/root/' ) def __init__(self): bt.Observer.__init__(self) if not os.path.isdir(self.p.rootpath): os.makedirs(self.p.rootpath) self._csvs:List[csv.DictWriter] = [] self._metadata = [] self._objects = self._owner.datas + list(self._owner.observers) for obj in self._objects: name = getTableName(obj) columns = getTableColumns(obj) path = os.path.join(self.p.rootpath,name+'.csv') metadata = (name,columns,open(path,'a'),) exists = os.path.isfile(path) csvfile = csv.DictWriter(metadata[2],[c for (c,_) in columns.items()]) if not exists: csvfile.writeheader() self._metadata.append(metadata) self._csvs.append(csvfile) def next(self): for (obj,meta,csv) in zip(self._objects,self._metadata,self._csvs): try: self._persist(obj,meta,csv) except: logging.exception("EXCEPTION OCCURRED WHILE WRITING TO CSV") def _persist(self,data:bt.DataBase,meta,csv:csv.DictWriter): name,columns = meta[0],meta[1] row = dict() foundValue = False for name,column in columns.items(): column:Column value = getattr(data.lines,name,None) if value is not None: if column == dt.datetime: value = data.datetime.datetime(0) # # Convert to UTC if data.contractdetails is not None: tz = pytz.timezone(data.contractdetails.m_timeZoneId) value = tz.normalize(tz.localize(value)).astimezone(pytz.utc) else: value = value[0] if type(value) == float and math.isnan(value): print("HI") continue foundValue = True row[name] = value if len(getattr(data,'datetime',[])) <= 1: data = self._owner.data0 value = data.datetime.datetime(0) if data.contractdetails is not None: tz = pytz.timezone(data.contractdetails.m_timeZoneId) value = tz.normalize(tz.localize(value)).astimezone(pytz.utc) row['datetime'] = value if foundValue: csv.writerow(row) def stop(self): for (name,columns,fd) in self._metadata: fd.flush() fd.close()

@andreyapopov said in Wrong execution price during the backtesting: I think another option would be to use YahooFinanceFeed with parameter adjclose=False, especially for things like SPY. It's too bad we can't choose to NOT adjust for dividends, but do adjust for splits. Yahoo is 100% broken. With quirks and random changes in what's the adjusted close across a data set. Unless you can identify a segment which is safe, don't use it. @andreyapopov said in Wrong execution price during the backtesting: Traceback (most recent call last): File "test.py", line 115, in <module> cerebro.run() File "/Library/Python/2.7/site-packages/backtrader/cerebro.py", line 1127, in run runstrat = self.runstrategies(iterstrat) File "/Library/Python/2.7/site-packages/backtrader/cerebro.py", line 1209, in runstrategies data.preload() File "/Library/Python/2.7/site-packages/backtrader/feed.py", line 692, in preload self.f.close() AttributeError: 'NoneType' object has no attribute 'close' It will be really difficult for @ab_trader (or anyone else) to identify what your problem is if you don't supply sample code. The error alone doesn't help.

@anfei I did something similar in my post. Putting it in a dictionary was an easy way to access it. You can see in the last comment here : link text

Or open a file with standard Python code and write out to it.

@sagittarius19 said in How to filter large stock universe then run trading system on small group: @backtrader thanks for the help regarding this. I know its prolly a very simple fix. Here is what I updated to, and im still getting an error. class Pandasclass(btfeeds.PandasData): lines = ('symbol') params = (('symbol', -1),) btfeeds.PandasData.datafields = btfeeds.PandasData.datafields + [u'symbol'] File "c:\Python27_path\Python27 Scritps\Extended pandas import attempt4.py", line 13, in <module> class Pandasclass(btfeeds.PandasData): File "c:\Python36\lib\site-packages\backtrader\lineseries.py", line 364, in __new__ linesoverride, lalias=la) File "c:\Python36\lib\site-packages\backtrader\lineseries.py", line 130, in _derive clslines = baselines + lines TypeError: can only concatenate tuple (not "str") to tuple When i put another dummy value next to it, I got this error. class Pandasclass(btfeeds.PandasData): lines = ('symbol','blank') params = (('symbol', -1),('blank', -1),) btfeeds.PandasData.datafields = btfeeds.PandasData.datafields + [u'symbol', u'blank'] File "c:\Python27_path\Python27 Scritps\Extended pandas import attempt4.py", line 49, in <module> cerebro.run() File "c:\Python36\lib\site-packages\backtrader\cerebro.py", line 1127, in run runstrat = self.runstrategies(iterstrat) File "c:\Python36\lib\site-packages\backtrader\cerebro.py", line 1209, in runstrategies data.preload() File "c:\Python36\lib\site-packages\backtrader\feed.py", line 435, in preload while self.load(): File "c:\Python36\lib\site-packages\backtrader\feed.py", line 476, in load _loadret = self._load() File "c:\Python36\lib\site-packages\backtrader\feeds\pandafeed.py", line 255, in _load line[0] = self.p.dataname.iloc[self._idx, colindex] File "c:\Python36\lib\site-packages\backtrader\linebuffer.py", line 222, in __setitem__ self.array[self.idx + ago] = value TypeError: must be real number, not str Does datas not allow for non numerical values to be in the columns? All values are floats. This is not a database.

@algoguy235 said in Detect new resampled price: I wasn't able to get len() as a line object in the init function len is a builtin and cannot be overridden. Given it is only meant to check a single value (because it increase monotonically) it seems pointless to have a replacement function for it to deliver a lines object.

@backtrader any suggestions?

Probably because you have a wrong understanding of what a Stop order is. It is not a stop-loss, it is a Stop, it may sell with a profit. When a trigger price is met, a Market order is executed. Unless you check the execution prices, both buy and sell it is pointless to make assumptions about a Trade being wrong. For the meaning of each order type, see: Docs - Orders Docs - Order Management and Execution

@aarzt93 said in Question regarding Fractal Indicator: But when I print it out later it gives me all the high prices instead of only those highs that were marked with a bearish fractal. @aarzt93 said in Question regarding Fractal Indicator: if self.fractal.fractal_bearish: self.__special_array_for_later.append(self.high[-2]) @backtrader said in Question regarding Fractal Indicator: Any value which is not an indication will be a NaN because there is nothing there. It was clearly stated that values which are not marked as fractal are NaN, so you should be specifically looking to skip NaN values. In any case the logic is flawed, because the mark for a fractal doesn't happen now, it happens two periods in the past, exactly as the high you want to store. @aarzt93 said in Question regarding Fractal Indicator: okay, so is there a way to store all the bearish/bullish fractals in a container (dict/array) and have them printed out, so that I can gather some statistics on them ? They are already stored in the indicator itself, which is a line. You probably want to read the documentation here: Docs - Platform Concepts and look for Slicing

Aaah got it thanks

@jmoons Thanks. I have experienced the same issue again after I added "bound off=1" to 15mins bar resampling. Not sure whether the parameter setting change is the direct cause though. Another suspicion is paper account. So far I just did testing using ib paper account and I've noticed the paper account server has different maintenance schedule from ib production server.

Thank you for point out the problem.

@backtrader Fantastic, thank you!

What an adventure. A lot of time spent and the solution is quite elegant. @backtrader was right of course; notify_timer was the key. Send me a PM if you would like the solution. It's rather simple, but ya, learned from reading the documentation more, reading what backtrader said and then some furthering of my python ability.

@backtrader ohoh I get it. My English understanding is really poor. Thanks for your patience.

I found a way to solve the error, in this thread: https://community.backtrader.com/topic/1043/dates-and-time-loaded-from-csv-files-not-precise/2

After some more working on it, I feel I am close to working through the problem (just have to wait for markets to open again on sunday night). When I run my latest script, I receive this error: TWS Time at connection:20190111 15:00:43 EST Traceback (most recent call last): File "D:/Dropbox/Parker Magic Dropbox/Elliot Parker/Drums/QuantStrategyProjectFiles/__init__ (1) (1) (1).py", line 222, in <module> run() File "D:/Dropbox/Parker Magic Dropbox/Elliot Parker/Drums/QuantStrategyProjectFiles/__init__ (1) (1) (1).py", line 219, in run cerebro.run() File "C:\Users\Elliot Parker\Anaconda3\lib\site-packages\backtrader\cerebro.py", line 1127, in run runstrat = self.runstrategies(iterstrat) 2018-12-12 AAPL-STK-SMART-USD File "C:\Users\Elliot Parker\Anaconda3\lib\site-packages\backtrader\cerebro.py", line 1298, in runstrategies not pos self._runnext(runstrats) tradetoday finish File "C:\Users\Elliot Parker\Anaconda3\lib\site-packages\backtrader\cerebro.py", line 1630, in _runnext strat._next() File "C:\Users\Elliot Parker\Anaconda3\lib\site-packages\backtrader\strategy.py", line 325, in _next super(Strategy, self)._next() File "C:\Users\Elliot Parker\Anaconda3\lib\site-packages\backtrader\lineiterator.py", line 270, in _next self.prenext() File "D:/Dropbox/Parker Magic Dropbox/Elliot Parker/Drums/QuantStrategyProjectFiles/__init__ (1) (1) (1).py", line 110, in prenext self.next() File "D:/Dropbox/Parker Magic Dropbox/Elliot Parker/Drums/QuantStrategyProjectFiles/__init__ (1) (1) (1).py", line 116, in next pos = self.getposition(d).size File "C:\Users\Elliot Parker\Anaconda3\lib\site-packages\backtrader\strategy.py", line 1366, in getposition return broker.getposition(data) File "C:\Users\Elliot Parker\Anaconda3\lib\site-packages\backtrader\brokers\ibbroker.py", line 306, in getposition return self.ib.getposition(data.tradecontract, clone=clone) File "C:\Users\Elliot Parker\Anaconda3\lib\site-packages\backtrader\lineseries.py", line 461, in __getattr__ return getattr(self.lines, name) AttributeError: 'Lines_LineSeries_DataSeries_OHLC_OHLCDateTime_Abst' object has no attribute 'tradecontract' So it is able to recognise the first datafeed, but is unable to recognize the second datafeed's name and date, this I am not sure how to solve. I might try cleaning up that file as so it only has one of each date, sorted, etc. My latest script: import datetime import backtrader as bt from http.cookiejar import _debug from Functions.PandasData_Extend import PandasData_Extend import pandas as pd import pytz # Create a Stratey class ADRLeader(bt.Strategy): StrategyInformation = dict( StrategyName='NextDayStrategyBinsS', Notes='US Leads AUS, BHP, Bin 0, Short| Freebandz', OrderExecutionType='Market', BuyLogic='%Bracket', SellLogic='%Bracket', DateRun=datetime.datetime.now(), DataGranularity='Day' ) params = dict( ma=bt.ind.SMA, p1=5, p2=15, limit=0.005, limdays=1, limdays2=1, hold=1, usebracket=False, # use order_target_size switchp1p2=False, # switch prices of order1 and order2 limu=0.005, limd=0.01, valid=1, when=bt.timer.SESSION_START, bin=0, ) def __init__(self): self.add_timer(when=self.p.when) self.orefs = list() self.order = None self.o = dict() self.signal = self.datas[1].PercentileDailyPctChange self.tradetoday = 0 def notify_data(self, data, status, *args, **kwargs): # print('*' * 5, 'DATA NOTIF:', data._getstatusname(status), *args) if status == data.LIVE: self.datastatus = 1 def notify_store(self, msg, *args, **kwargs): # print('*' * 5, 'STORE NOTIF:', msg) return def notify_order(self, order): if order.status == order.Completed: # self.holdstart = len(self) # print('Order Completed') # print(self.datas[0].datetime.datetime()) # print(order.price) return if order.status == order.Accepted: # print('Order Accepted') return if not order.alive() and order.ref in self.orefs: self.orefs.remove(order.ref) if order.isbuy(): if order.status == order.Submitted: # print('Order Submitted BUY') # print(order.price) # print(str(order.price) + 'Order Buy Price') return if order.issell(): if order.status == order.Submitted: # print('Order Submitted SELL') # print(order.price) return def notify_trade(self, trade): if trade.isclosed: # print('tradeclosed') # date = self.datas[0].datetime.datetime() # print(self.datas[0].datetime.datetime()) # print(date, trade.pnl, trade.pnlcomm) # tradetoday = (bt.num2date(trade.dtopen)).date() # print(trade.ref) # self.listforall.append(tradetoday) # print('tradetoday') # print(tradetoday) self.tradetoday = 1 if trade.isopen: # print('tradeopen') # date = self.datas[0].datetime.datetime() # time = self.datas[0].datetime.time(0) # print(date, trade.price) return def notify_timer(self, timer, when): # print(str(when.time()) +' '+' juice') if when.time() == datetime.time(0,0): # print('Its a new day') # print(self.datas[0].datetime.datetime()) self.tradetoday = 0 else: # print('does not') return def prenext(self): self.next() def next(self): for i, d in enumerate(self.datas): dt, dn = self.datetime.date(), d._name pos = self.getposition(d).size # print(AAPL-STK-SMART-USD) print('{} {}'.format(dt, dn)) # print(i[0]) #Setting up the Not in Position if i == 0: pos = self.getposition(d).size # print('position size start') # print(pos) # print('position size finish') else: pass if pos == 0: print('not pos') #Setting up the If Not traded today if not self.tradetoday == 1: # print('tradetoday start') # print(self.tradetoday) print('tradetoday finish') if (len(self.datas[i]) > 0): # print(self.datetime.date()) #print(self.signal) print('top') # print(self.data0.Close) print('bottom') # print (self.data0.l.close[-1]) if d._dataname.PercentileDailyPctChange[+1] == 12: print('PercentileValue') print(d.datetime.datetime().isoformat()) print(d._dataname.PercentileDailyPctChange[+1]) print('PercentileValueFinish') if i == 0: print(d.close[0]) # self.buy(data=d, ) elif i == 0: print('0') print('Currency Latest Close') print(self.datas[0].datetime.datetime().isoformat()) print(d.close[0]) print('Currency Latest Finish') else: print('nope') return else: print('In Position') return else: print('no datasets coming in') return def start(self): if self.data0.contractdetails is not None: # print('yo') # print('Timezone from ContractDetails: {}'.format( # self.data0.contractdetails.m_timeZoneId)) return header = ['Datetime', 'Open', 'High', 'Low', 'Close'] # print(', '.join(header)) self.done = False def run(args=None): cerebro = bt.Cerebro() storekwargs = dict( host='127.0.0.1', port=7496, clientId=None, reconnect=100, timeout=10.0, _debug=False) ibstore = bt.stores.IBStore(**storekwargs) broker = ibstore.getbroker() cerebro.setbroker(broker) # 'EUR.USD-CASH-IDEALPRO' # AAPL-STK-SMART-USD # , tradename='AAPL-CFD-SMART-USD' # , tradename = 'BHP-CFD-SMART-AUD' # BHP-STK-SMART-AUD data0 = ibstore.getdata(reqRealTimeBars=True, dataname='AAPL-STK-SMART-USD', rtbar=False, timeframe=bt.TimeFrame.Seconds, compression=1) # cerebro.resampledata(data0, timeframe=bt.TimeFrame.Seconds, # compression=3) cerebro.adddata(data0) # print('remix') # print(data0) # cerebro.adddata(data0, name='d0') StockPercentile = pd.read_csv(r'D:\Dropbox\Parker Magic Dropbox\Elliot Parker\Drums\QuantStrategyResults\AUSLeadsUSPercentile.csv', header=0, index_col=0, parse_dates=True) StockPercentile = StockPercentile[~StockPercentile.index.duplicated(keep='first')] StockPercentile = StockPercentile.sort_index(ascending=False) StockPercentile = pd.DataFrame(StockPercentile.iloc[:, :]) # print(StockPercentile) data1 = PandasData_Extend(dataname=StockPercentile, timeframe=bt.TimeFrame.Days, compression=1) # cerebro.resampledata(data1) # cerebro.replaydata(data1) cerebro.adddata(data1) cerebro.addstrategy(ADRLeader) cerebro.run() if __name__ == '__main__': run()

I was able to find a solution, self.data.close.get(size=1, ago=-1)[0] works right as I want

@noobtrader nothing in your post shows that you are using live data. So why do you expect to have it in the answer?