CSV data Duplication

amit

Hello, I have downloaded the 12h data using CCXT. When I import the data using CSV reader into cerebero, for each date the next function records two calls at 12:00 PM like below, but when I use CCXT live feed from exchange the same data is written only once in the next function. Because of this the results of the same strategy are different when connected to exchange feed compared to csv file. Please help.

This is what is feed as CSV data, I used unixtimestamp as well but that didnt help

2019-01-09 00:00:00,3955.45,4006.81,3930.04,3986.44,14463.608386
2019-01-09 12:00:00,3985.52,3996.3,3941.0,3966.65,14525.831125
2019-01-10 00:00:00,3966.06,3996.01,3715.65,3739.99,31791.868686
2019-01-10 12:00:00,3738.01,3750.0,3540.0,3585.88,27610.359824
2019-01-11 00:00:00,3585.88,3635.0,3546.36,3594.0,18501.371276
2019-01-11 12:00:00,3593.57,3658.0,3465.0,3601.31,19837.283457
2019-01-12 00:00:00,3601.31,3618.19,3533.0,3571.16,10429.103694
2019-01-12 12:00:00,3571.89,3612.39,3530.0,3583.13,11570.82466

This is what is written out of next function
2019-01-09 00:00:00,3955.45,4006.81,3930.04
2019-01-09 12:00:00,3985.52,3996.3,3941.0
2019-01-09 12:00:00,3985.52,3996.3,3941.0
2019-01-10 00:00:00,3966.06,3996.01,3715.65
2019-01-10 12:00:00,3738.01,3750.0,3540.0
2019-01-10 12:00:00,3738.01,3750.0,3540.0
2019-01-11 00:00:00,3585.88,3635.0,3546.36
2019-01-11 12:00:00,3593.57,3658.0,3465.0
2019-01-11 12:00:00,3593.57,3658.0,3465.0
2019-01-12 00:00:00,3601.31,3618.19,3533.0
2019-01-12 12:00:00,3571.89,3612.39,3530.0
2019-01-12 12:00:00,3571.89,3612.39,3530.0

run-out

@amit Without your code it's impossble to help you.

amit

@run-out Apologies... below is the code for loading the data using the CSV, and this produces duplicate results.

class customCSV12h(btfeed.GenericCSVData):
params = (
('dtformat', '%Y-%m-%d %H:%M:%S'),
('timeframe',bt.TimeFrame.Minutes),
('compression',720),
('datetime', 0),
('open', 1),
('high', 2),
('low', 3),
('close', 4),
('volume', 5),
('openinterest', -1),
)

class customCSV1d(btfeed.GenericCSVData):
params = (
('dtformat', '%Y-%m-%d'),
('timeframe',bt.TimeFrame.Days),
('compression',1),
('datetime', 0),
('open', 1),
('high', 2),
('low', 3),
('close', 4),
('volume', 5),
('openinterest', -1),
)

data12h = customCSV12h(dataname='binance-BTCUSDT-12h.csv')
data1d = customCSV1d(dataname='binance-BTCUSDT-1d.csv')
cerebro.adddata(data12h)
cerebro.adddata(data1d)

amit

@run-out and this is the strategy file...

from future import (
absolute_import,
division,
print_function,
unicode_literals,
)

import datetime
import os
import sys
import EventLogger as logger
import backtrader as bt
from models import LineMemory

class customStrategy(bt.Strategy):

average_period = 15
total_trades = 0
wins = 0
losses = 0
gross_profits = 0
gross_losses = 0
percent_profitable = 0
profit_factor = 0

high_10_count = 0
high_15_count = 0

low_10_count = 0
low_15_count = 0

will_short_close = False
will_short_close_10 = False
will_short_close_15 = False
will_long_close = False
will_long_close_10 = False
will_long_close_15 = False

def __init__(self):
    self.highs = []
    self.lows = []
    self.highs.append(LineMemory(10))
    self.highs.append(LineMemory(15))
    self.highs.append(LineMemory(5))
    self.lows.append(LineMemory(5))
    self.lows.append(LineMemory(10))
    self.lows.append(LineMemory(15))

    self.dataclose = self.datas[0].close
    self.order = None
    self.buyprice = None
    self.buycomm = None # again optional

    # Indicators
    self.sma = bt.indicators.SimpleMovingAverage(
        self.datas[0], period=self.average_period
    )

def log(self, msg, dt=None):
    dt = dt or self.datas[0].datetime.date(0)
    print("{}, {}".format(dt.isoformat(), msg))

def notify_order(self, order):
    if order.status in [order.Submitted, order.Accepted]:
        return

    if order.status in [order.Completed]:
        if order.isbuy():
            self.log(
                "BUY, price: {}, cost: {}, comm: {}".format(
                    order.executed.price,
                    order.executed.value,
                    order.executed.comm,
                )
            )
            self.buyprice = order.executed.price
            self.buycomm = order.executed.comm
        elif order.issell():
            self.log(
                "SELL, price: {}, cost: {}, comm: {}".format(
                    order.executed.price,
                    order.executed.value,
                    order.executed.comm,
                )
            )
        self.bar_executed = len(self)
    elif order.status in [order.Canceled, order.Margin, order.Rejected]:
        self.log("Order Canceled/Margin/Rejected")

    self.order = None

def notify_trade(self, trade):
    if not trade.isclosed:
        return

    if trade.pnl > 0:
        self.wins += 1
        self.gross_profits += trade.pnl
    else:
        self.losses += 1
        self.gross_losses -= trade.pnl
    
    if(self.total_trades > 0):
        self.percent_profitable = self.wins / self.total_trades

    if(self.gross_losses>0):
        self.profit_factor = self.gross_profits / self.gross_losses

    self.log("PROFIT, gross: {}, net: {}".format(trade.pnl, trade.pnlcomm))

def next(self):
    
    self.log1.logInfo("{},{},{},{} ".format(
        str(self.datas[0].datetime.date(0))+ ' ' +str(self.datas[0].datetime.time(0)),
        self.data_open[0],
        self.data_high[0],
        self.data_low[0],
        self.dataclose[0]
    ))
    
    for item in self.highs:
        item.push(self.data_high[0]) # self.dataclose[0]
    for item in self.lows:
        item.push(self.data_low[0]) # self.dataclose[0]

    if (
        self.high_10_count + self.high_15_count > 2
        or self.high_10_count + self.high_15_count < 0
    ):
        print("Bonkers Long")
    if (
        self.low_10_count + self.low_15_count > 2
        or self.low_10_count + self.low_15_count < 0
    ):
        print("Bonkers Short)

    self.will_short_close = False
    self.will_short_close_10 = False
    self.will_short_close_15 = False
    self.will_long_close = False
    self.will_long_close_10 = False
    self.will_long_close_15 = False
    
    if self.order:
        return

    if len(self) > 10:
        if (
            self.high_10_count
            and not self.high_15_count
            and self.data_low[0] < self.lows[0].lowest
        ):
            self.will_long_close = True
            self.will_long_close_10 = True

        if (
            self.low_10_count
            and not self.low_15_count
            and self.data_high[0] > self.highs[2].highest
        ):
            self.will_short_close = True
            self.will_short_close_20 = True

        if (
            self.data_high[0] > self.highs[0].highest
            and not self.high_10_count
        ):
            self.high_10_count += 1
            self.log("LONG, {} 10 day entry".format(self.dataclose[0]))
            self.order = self.buy()

        if self.will_long_close_10:
            self.total_trades += 1
            self.high_10_count = 0
            self.high_15_count = 0
            self.log("LONG CLOSE, {}  5 day low".format(self.dataclose[0]))
            self.order = self.close()

        if len(self) > 15:
            if (
                self.high_15_count
                and self.dataclose[0] < self.lows[1].lowest
            ):
                self.will_long_close = True
                self.will_long_close_15 = True

            if (
                self.low_15_count
                and self.data_high[0] > self.highs[0].highest
            ):
                self.will_short_close = True
                self.will_short_close_15 = True

            if (
                self.data_high[0] > self.highs[1].highest
                and not self.high_15_count
            ):
                if self.position.size < 0 and not self.will_short_close:
                    self.order = self.close()
                self.high_15_count += 1
                self.log("LONG, {} 15 day entry".format(self.dataclose[0]))
                self.order = self.buy()

            if self.will_long_close_15:
                self.total_trades += 1
                self.high_10_count = 0
                self.high_15_count = 0
                self.log("LONG CLOSE, {} 10 day low".format(self.dataclose[0]))
                self.order = self.close()

        if (
            self.data_low[0] < self.lows[1].lowest
            and not self.low_10_count
        ):
            self.low_10_count += 1
            self.log("SHORT, {} 10 day low".format(self.dataclose[0]))
            self.order = self.sell()

        if self.will_short_close_10:
            self.total_trades += 1
            self.low_10_count = 0
            self.low_15_count = 0
            self.log("SHORT CLOSE, {} 10 day high".format(self.dataclose[0]))
            self.order = self.close()
        if len(self) > 15:
            if (
                self.data_low[0] < self.lows[2].lowest 
                and not self.low_15_count
            ):
                if self.position.size > 0 and not self.will_long_close:
                    self.order = self.close()
                self.low_15_count += 1
                self.log("SHORT, {} 15 day low".format(self.dataclose[0]))
                self.order = self.sell()

            if self.will_short_close_15:
                self.total_trades += 1
                self.low_10_count = 0
                self.low_15_count = 0
                self.log("SHORT CLOSE, {} 10 day high".format(self.dataclose[0]))
                self.order = self.close()

def stop(self):
    print("Total Trades: {}".format(self.total_trades))
    print("Percent Profitable: {}".format(self.percent_profitable))
    print("Profit Factor: {}".format(self.profit_factor))
    super().stop()

amit

@run-out and if I use the live feed from Binance like below I get correct results.

cerebro.adddata(CCXTFeed(exchange='binance',
dataname='BTC/USDT',
timeframe=bt.TimeFrame.Minutes,
fromdate=dt.datetime(2020, 1, 1, 0, 0),
todate=dt.datetime(2021, 3, 2, 0, 0),
compression=720,
ohlcv_limit=512,
currency='BTC',
retries=5,

# 'apiKey' and 'secret' are skipped
config=broker_config))

cerebro.adddata(CCXTFeed(exchange='binance',
dataname='BTC/USDT',
timeframe=bt.TimeFrame.Days,
fromdate=dt.datetime(2020, 1, 1, 0, 0),
todate=dt.datetime(2021, 3, 2, 0, 0),
compression=1,
ohlcv_limit=512,
currency='BTC',
retries=5,

# 'apiKey' and 'secret' are skipped
config=broker_config))

run-out

@amit Have a look at the answer to question one here.

Let us know if that works. Thanks.