CSV data Duplication
-
Hello, I have downloaded the 12h data using CCXT. When I import the data using CSV reader into cerebero, for each date the next function records two calls at 12:00 PM like below, but when I use CCXT live feed from exchange the same data is written only once in the next function. Because of this the results of the same strategy are different when connected to exchange feed compared to csv file. Please help.
This is what is feed as CSV data, I used unixtimestamp as well but that didnt help
2019-01-09 00:00:00,3955.45,4006.81,3930.04,3986.44,14463.608386
2019-01-09 12:00:00,3985.52,3996.3,3941.0,3966.65,14525.831125
2019-01-10 00:00:00,3966.06,3996.01,3715.65,3739.99,31791.868686
2019-01-10 12:00:00,3738.01,3750.0,3540.0,3585.88,27610.359824
2019-01-11 00:00:00,3585.88,3635.0,3546.36,3594.0,18501.371276
2019-01-11 12:00:00,3593.57,3658.0,3465.0,3601.31,19837.283457
2019-01-12 00:00:00,3601.31,3618.19,3533.0,3571.16,10429.103694
2019-01-12 12:00:00,3571.89,3612.39,3530.0,3583.13,11570.82466This is what is written out of next function
2019-01-09 00:00:00,3955.45,4006.81,3930.04
2019-01-09 12:00:00,3985.52,3996.3,3941.0
2019-01-09 12:00:00,3985.52,3996.3,3941.0
2019-01-10 00:00:00,3966.06,3996.01,3715.65
2019-01-10 12:00:00,3738.01,3750.0,3540.0
2019-01-10 12:00:00,3738.01,3750.0,3540.0
2019-01-11 00:00:00,3585.88,3635.0,3546.36
2019-01-11 12:00:00,3593.57,3658.0,3465.0
2019-01-11 12:00:00,3593.57,3658.0,3465.0
2019-01-12 00:00:00,3601.31,3618.19,3533.0
2019-01-12 12:00:00,3571.89,3612.39,3530.0
2019-01-12 12:00:00,3571.89,3612.39,3530.0 -
@amit Without your code it's impossble to help you.
-
@run-out Apologies... below is the code for loading the data using the CSV, and this produces duplicate results.
class customCSV12h(btfeed.GenericCSVData):
params = (
('dtformat', '%Y-%m-%d %H:%M:%S'),
('timeframe',bt.TimeFrame.Minutes),
('compression',720),
('datetime', 0),
('open', 1),
('high', 2),
('low', 3),
('close', 4),
('volume', 5),
('openinterest', -1),
)class customCSV1d(btfeed.GenericCSVData):
params = (
('dtformat', '%Y-%m-%d'),
('timeframe',bt.TimeFrame.Days),
('compression',1),
('datetime', 0),
('open', 1),
('high', 2),
('low', 3),
('close', 4),
('volume', 5),
('openinterest', -1),
)data12h = customCSV12h(dataname='binance-BTCUSDT-12h.csv')
data1d = customCSV1d(dataname='binance-BTCUSDT-1d.csv')
cerebro.adddata(data12h)
cerebro.adddata(data1d) -
@run-out and this is the strategy file...
from future import (
absolute_import,
division,
print_function,
unicode_literals,
)import datetime
import os
import sys
import EventLogger as logger
import backtrader as bt
from models import LineMemoryclass customStrategy(bt.Strategy):
average_period = 15 total_trades = 0 wins = 0 losses = 0 gross_profits = 0 gross_losses = 0 percent_profitable = 0 profit_factor = 0 high_10_count = 0 high_15_count = 0 low_10_count = 0 low_15_count = 0 will_short_close = False will_short_close_10 = False will_short_close_15 = False will_long_close = False will_long_close_10 = False will_long_close_15 = False def __init__(self): self.highs = [] self.lows = [] self.highs.append(LineMemory(10)) self.highs.append(LineMemory(15)) self.highs.append(LineMemory(5)) self.lows.append(LineMemory(5)) self.lows.append(LineMemory(10)) self.lows.append(LineMemory(15)) self.dataclose = self.datas[0].close self.order = None self.buyprice = None self.buycomm = None # again optional # Indicators self.sma = bt.indicators.SimpleMovingAverage( self.datas[0], period=self.average_period ) def log(self, msg, dt=None): dt = dt or self.datas[0].datetime.date(0) print("{}, {}".format(dt.isoformat(), msg)) def notify_order(self, order): if order.status in [order.Submitted, order.Accepted]: return if order.status in [order.Completed]: if order.isbuy(): self.log( "BUY, price: {}, cost: {}, comm: {}".format( order.executed.price, order.executed.value, order.executed.comm, ) ) self.buyprice = order.executed.price self.buycomm = order.executed.comm elif order.issell(): self.log( "SELL, price: {}, cost: {}, comm: {}".format( order.executed.price, order.executed.value, order.executed.comm, ) ) self.bar_executed = len(self) elif order.status in [order.Canceled, order.Margin, order.Rejected]: self.log("Order Canceled/Margin/Rejected") self.order = None def notify_trade(self, trade): if not trade.isclosed: return if trade.pnl > 0: self.wins += 1 self.gross_profits += trade.pnl else: self.losses += 1 self.gross_losses -= trade.pnl if(self.total_trades > 0): self.percent_profitable = self.wins / self.total_trades if(self.gross_losses>0): self.profit_factor = self.gross_profits / self.gross_losses self.log("PROFIT, gross: {}, net: {}".format(trade.pnl, trade.pnlcomm)) def next(self): self.log1.logInfo("{},{},{},{} ".format( str(self.datas[0].datetime.date(0))+ ' ' +str(self.datas[0].datetime.time(0)), self.data_open[0], self.data_high[0], self.data_low[0], self.dataclose[0] )) for item in self.highs: item.push(self.data_high[0]) # self.dataclose[0] for item in self.lows: item.push(self.data_low[0]) # self.dataclose[0] if ( self.high_10_count + self.high_15_count > 2 or self.high_10_count + self.high_15_count < 0 ): print("Bonkers Long") if ( self.low_10_count + self.low_15_count > 2 or self.low_10_count + self.low_15_count < 0 ): print("Bonkers Short) self.will_short_close = False self.will_short_close_10 = False self.will_short_close_15 = False self.will_long_close = False self.will_long_close_10 = False self.will_long_close_15 = False if self.order: return if len(self) > 10: if ( self.high_10_count and not self.high_15_count and self.data_low[0] < self.lows[0].lowest ): self.will_long_close = True self.will_long_close_10 = True if ( self.low_10_count and not self.low_15_count and self.data_high[0] > self.highs[2].highest ): self.will_short_close = True self.will_short_close_20 = True if ( self.data_high[0] > self.highs[0].highest and not self.high_10_count ): self.high_10_count += 1 self.log("LONG, {} 10 day entry".format(self.dataclose[0])) self.order = self.buy() if self.will_long_close_10: self.total_trades += 1 self.high_10_count = 0 self.high_15_count = 0 self.log("LONG CLOSE, {} 5 day low".format(self.dataclose[0])) self.order = self.close() if len(self) > 15: if ( self.high_15_count and self.dataclose[0] < self.lows[1].lowest ): self.will_long_close = True self.will_long_close_15 = True if ( self.low_15_count and self.data_high[0] > self.highs[0].highest ): self.will_short_close = True self.will_short_close_15 = True if ( self.data_high[0] > self.highs[1].highest and not self.high_15_count ): if self.position.size < 0 and not self.will_short_close: self.order = self.close() self.high_15_count += 1 self.log("LONG, {} 15 day entry".format(self.dataclose[0])) self.order = self.buy() if self.will_long_close_15: self.total_trades += 1 self.high_10_count = 0 self.high_15_count = 0 self.log("LONG CLOSE, {} 10 day low".format(self.dataclose[0])) self.order = self.close() if ( self.data_low[0] < self.lows[1].lowest and not self.low_10_count ): self.low_10_count += 1 self.log("SHORT, {} 10 day low".format(self.dataclose[0])) self.order = self.sell() if self.will_short_close_10: self.total_trades += 1 self.low_10_count = 0 self.low_15_count = 0 self.log("SHORT CLOSE, {} 10 day high".format(self.dataclose[0])) self.order = self.close() if len(self) > 15: if ( self.data_low[0] < self.lows[2].lowest and not self.low_15_count ): if self.position.size > 0 and not self.will_long_close: self.order = self.close() self.low_15_count += 1 self.log("SHORT, {} 15 day low".format(self.dataclose[0])) self.order = self.sell() if self.will_short_close_15: self.total_trades += 1 self.low_10_count = 0 self.low_15_count = 0 self.log("SHORT CLOSE, {} 10 day high".format(self.dataclose[0])) self.order = self.close() def stop(self): print("Total Trades: {}".format(self.total_trades)) print("Percent Profitable: {}".format(self.percent_profitable)) print("Profit Factor: {}".format(self.profit_factor)) super().stop()
-
@run-out and if I use the live feed from Binance like below I get correct results.
cerebro.adddata(CCXTFeed(exchange='binance',
dataname='BTC/USDT',
timeframe=bt.TimeFrame.Minutes,
fromdate=dt.datetime(2020, 1, 1, 0, 0),
todate=dt.datetime(2021, 3, 2, 0, 0),
compression=720,
ohlcv_limit=512,
currency='BTC',
retries=5,# 'apiKey' and 'secret' are skipped config=broker_config))
cerebro.adddata(CCXTFeed(exchange='binance',
dataname='BTC/USDT',
timeframe=bt.TimeFrame.Days,
fromdate=dt.datetime(2020, 1, 1, 0, 0),
todate=dt.datetime(2021, 3, 2, 0, 0),
compression=1,
ohlcv_limit=512,
currency='BTC',
retries=5,# 'apiKey' and 'secret' are skipped config=broker_config))
-