Resample live feed from CCXT problem
-
Guys,
I'm trying to get macd of 1minute and 5minute timeframe.
But I'm gettingValueError: min() arg is an empty sequence
error.Here's the script:
# !/usr/bin/env python # -*- coding: utf-8; py-indent-offset:4 -*- from __future__ import (absolute_import, division, print_function, unicode_literals) import sys import time from datetime import datetime, timedelta from datetime import datetime, timedelta import backtrader as bt import ccxt class TestStrategy(bt.Strategy): params = ( ('printlog', True), ) def log(self, txt, dt=None, doprint=False): ''' Logging function fot this strategy''' if self.params.printlog or doprint: dt = dt or bt.num2date(self.data.datetime[0]) print('%s, %s' % (dt, txt)) def start(self): self.counter = 0 print('START') def prenext(self): self.counter += 1 print('prenext len %d - counter %d' % (len(self), self.counter)) self.log("data#0: " + str(self.datas[0].datetime[0])) self.log("data#1: " + str(self.datas[1].datetime[0])) def __init__(self): self.macd = bt.indicators.MACDHisto(self.datas[0]) self.macd2 = bt.indicators.MACDHisto(self.datas[1]) def next(self): self.counter += 1 price_txt = "Counter: " + str(self.counter) + " Open/Close/High/Low/Volume: " + str(self.data0.open[0]) + " - "+ str(self.data0.close[0]) + " - " + str(self.data0.high[0]) + " - " + str(self.data0.low[0]) + " - " + str(self.data0.volume[0]) + " Len: "+ str(len(self.data0))# + " Time Frame:" + bt.TimeFrame.getname(self.data0._timeframe) + " Len: "+ str(len(self.data0)) self.log(price_txt) macd_txt = "MACD: {:.2f}, Histo: {:.2f}".format(self.macd.macd[0],self.macd.histo[0]) self.log("MACD#1: " + macd_txt) macd2_txt = "MACD: {:.2f}, Histo: {:.2f}".format(self.macd2.macd[0],self.macd2.histo[0]) self.log("MACD#2: " + macd2_txt) if __name__ == '__main__': cerebro = bt.Cerebro() #exchange = sys.argv[1] if len(sys.argv) > 1 else 'gdax' exchange = sys.argv[1] if len(sys.argv) > 1 else 'okex' symbol = sys.argv[2] if len(sys.argv) > 2 else 'ETH/USDT' hist_start_date = datetime.utcnow() - timedelta(minutes=10) print('UTC NOW: ', datetime.utcnow()) print('hist_start_data: ', hist_start_date) print('Using symbol: ', symbol) # Create data feeds data_1m = bt.feeds.CCXT(exchange=exchange, symbol=symbol, name="1m", timeframe=bt.TimeFrame.Minutes, fromdate=hist_start_date,compression=1) cerebro.adddata(data_1m) cerebro.replaydata(data_1m, timeframe=bt.TimeFrame.Minutes, name="2m",compression=2) cerebro.addstrategy(TestStrategy) cerebro.run()
And the output:
$ python stage1.py UTC NOW: 2019-06-21 08:33:35.360573 hist_start_data: 2019-06-21 08:23:35.360331 Using symbol: ETH/USDT START dts[]: [737231.35, 737231.35] prenext len 1 - counter 1 2019-06-21 08:24:00, data#0: 737231.35 2019-06-21 08:24:00, data#1: 737231.35 dts[]: [737231.3506944445, None] prenext len 2 - counter 2 2019-06-21 08:25:00, data#0: 737231.3506944445 2019-06-21 08:25:00, data#1: 737231.35 dts[]: [737231.3513888889, 737231.3513888889] prenext len 3 - counter 3 2019-06-21 08:26:00, data#0: 737231.3513888889 2019-06-21 08:26:00, data#1: 737231.3513888889 dts[]: [737231.3520833333, None] prenext len 4 - counter 4 2019-06-21 08:27:00, data#0: 737231.3520833333 2019-06-21 08:27:00, data#1: 737231.3513888889 dts[]: [737231.3527777778, 737231.3527777778] prenext len 5 - counter 5 2019-06-21 08:28:00, data#0: 737231.3527777778 2019-06-21 08:28:00, data#1: 737231.3527777778 dts[]: [737231.3534722222, None] prenext len 6 - counter 6 2019-06-21 08:29:00, data#0: 737231.3534722222 2019-06-21 08:29:00, data#1: 737231.3527777778 dts[]: [737231.3541666666, 737231.3541666666] prenext len 7 - counter 7 2019-06-21 08:30:00, data#0: 737231.3541666666 2019-06-21 08:30:00, data#1: 737231.3541666666 dts[]: [737231.3548611111, None] prenext len 8 - counter 8 2019-06-21 08:31:00, data#0: 737231.3548611111 2019-06-21 08:31:00, data#1: 737231.3541666666 dts[]: [737231.3555555556, 737231.3555555556] prenext len 9 - counter 9 2019-06-21 08:32:00, data#0: 737231.3555555556 2019-06-21 08:32:00, data#1: 737231.3555555556 dts[]: [737231.35625, None] prenext len 10 - counter 10 2019-06-21 08:33:00, data#0: 737231.35625 2019-06-21 08:33:00, data#1: 737231.3555555556 dts[]: [None, 737231.3569444445] Traceback (most recent call last): File "stage1.py", line 69, in <module> cerebro.run() File "/Users/michael/.venv/backtrader-ccxt/lib/python3.6/site-packages/backtrader/cerebro.py", line 1127, in run runstrat = self.runstrategies(iterstrat) File "/Users/michael/.venv/backtrader-ccxt/lib/python3.6/site-packages/backtrader/cerebro.py", line 1298, in runstrategies self._runnext(runstrats) File "/Users/michael/.venv/backtrader-ccxt/lib/python3.6/site-packages/backtrader/cerebro.py", line 1558, in _runnext dt0 = min((d for i, d in enumerate(dts) ValueError: min() arg is an empty sequence
You may not see the below:
2019-06-21 08:32:00, data#0: 737231.3555555556 2019-06-21 08:32:00, data#1: 737231.3555555556 dts[]: [737231.35625, None]
because I changed the source code to print them out.
Any ideas?
What am I doing wrong?
Any help appreciated!
-
This seems related to your usage of the
CCXT
fork, as per your other message:The fork will have to handle it.
-
@backtrader Hi Backtrader, I guess not.
Because I triedbt-ccxt-store
too. It's the same error.And it's going well if remove the
fromdate
. And also, leads to a before time point, don't know why.Here's goes the
bt-ccxt-store
version script, which is almost the same:# !/usr/bin/env python # -*- coding: utf-8; py-indent-offset:4 -*- from __future__ import (absolute_import, division, print_function, unicode_literals) import sys import time from datetime import datetime, timedelta import backtrader as bt from ccxtbt import CCXTFeed class TestStrategy(bt.Strategy): params = ( ('printlog', True), ) def log(self, txt, dt=None, doprint=False): ''' Logging function fot this strategy''' if self.params.printlog or doprint: dt = dt or bt.num2date(self.data.datetime[0]) print('%s, %s' % (dt, txt)) def start(self): self.counter = 0 print('START') def prenext(self): self.counter += 1 print('prenext len %d - counter %d' % (len(self), self.counter)) def __init__(self): self.macd = bt.indicators.MACDHisto(self.datas[0]) self.macd2 = bt.indicators.MACDHisto(self.datas[1]) def next(self): self.counter += 1 price_txt = "Counter: " + str(self.counter) + " Open/Close/High/Low/Volume: " + str(self.data0.open[0]) + " - "+ str(self.data0.close[0]) + " - " + str(self.data0.high[0]) + " - " + str(self.data0.low[0]) + " - " + str(self.data0.volume[0]) + " Len: "+ str(len(self.data0))# + " Time Frame:" + bt.TimeFrame.getname(self.data0._timeframe) + " Len: "+ str(len(self.data0)) self.log(price_txt) macd_txt = "MACD: {:.2f}, Histo: {:.2f}".format(self.macd.macd[0],self.macd.histo[0]) self.log("MACD#1: " + macd_txt) macd2_txt = "MACD: {:.2f}, Histo: {:.2f}".format(self.macd2.macd[0],self.macd2.histo[0]) self.log("MACD#2: " + macd2_txt) def notify_data(self, data, status, *args, **kwargs): dn = data._name dt = datetime.now() msg= 'Data Status: {}'.format(data._getstatusname(status)) print(dt,dn,msg) if data._getstatusname(status) == 'LIVE': self.live_data = True else: self.live_data = False if __name__ == '__main__': #cerebro = bt.Cerebro(quicknotify=True) cerebro = bt.Cerebro() #exchange = sys.argv[1] if len(sys.argv) > 1 else 'gdax' exchange = sys.argv[1] if len(sys.argv) > 1 else 'binance' symbol = sys.argv[2] if len(sys.argv) > 2 else 'ETH/USDT' #store = CCXTStore(exchange=exchange, currency='LTC', config=config, retries=5, debug=False) hist_start_date = datetime.utcnow() - timedelta(minutes=10) print('UTC NOW: ', datetime.utcnow()) print('hist_start_data: ', hist_start_date) print('Using symbol: ', symbol) data = CCXTFeed(exchange=exchange, dataname=symbol, timeframe=bt.TimeFrame.Minutes, #fromdate=hist_start_date, #todate=datetime(2019, 1, 1, 0, 2), compression=1, ohlcv_limit=2, currency='USDT', retries=5, config={'enableRateLimit': True, 'nonce': lambda: str(int(time.time() * 1000))}) cerebro.adddata(data) cerebro.resampledata(data, timeframe=bt.TimeFrame.Minutes, compression=5) cerebro.addstrategy(TestStrategy) # Run the strategy cerebro.run()
With the following output:
prenext len 47 - counter 47 prenext len 48 - counter 48 prenext len 49 - counter 49 Traceback (most recent call last): File "ccxtbttest.py", line 88, in <module> cerebro.adddata(data) File "/Users/michael/.venv/bt-ccxt-store/lib/python3.6/site-packages/backtrader-1.9.74.123-py3.6.egg/backtrader/cerebro.py", line 1127, in run runstrat = self.runstrategies(iterstrat) File "/Users/michael/.venv/bt-ccxt-store/lib/python3.6/site-packages/backtrader-1.9.74.123-py3.6.egg/backtrader/cerebro.py", line 1298, in runstrategies self._runnext(runstrats) File "/Users/michael/.venv/bt-ccxt-store/lib/python3.6/site-packages/backtrader-1.9.74.123-py3.6.egg/backtrader/cerebro.py", line 1557, in _runnext dt0 = min((d for i, d in enumerate(dts) ValueError: min() arg is an empty sequence (bt-ccxt-store)
And also Another script:
from ccxtbt import CCXTStore import backtrader as bt from datetime import datetime, timedelta class TestStrategy(bt.Strategy): def __init__(self): self.sma = bt.indicators.SMA(self.datas[0],period=21) #self.macd5 = bt.indicators.MACD(self.datas[1]) def next(self): # Get cash and balance # New broker method that will let you get the cash and balance for # any wallet. It also means we can disable the getcash() and getvalue() # rest calls before and after next which slows things down. # NOTE: If you try to get the wallet balance from a wallet you have # never funded, a KeyError will be raised! Change LTC below as approriate if self.live_data: cash, value = self.broker.get_wallet_balance('USDT') else: # Avoid checking the balance during a backfill. Otherwise, it will # Slow things down. cash = 'NA' for data in self.datas: print('{} - {} \t| Cash {} | O: {} H: {} L: {} C: {} V:{} \tSMA:{}'.format(data.datetime.datetime(), data._name, cash, data.open[0], data.high[0], data.low[0], data.close[0], data.volume[0], self.sma[0])) def notify_data(self, data, status, *args, **kwargs): dn = data._name dt = datetime.now() msg= 'Data Status: {}'.format(data._getstatusname(status)) print(dt,dn,msg) if data._getstatusname(status) == 'LIVE': self.live_data = True else: self.live_data = False apikey = 'x' secret = 'xx' cerebro = bt.Cerebro(quicknotify=True) # Add the strategy cerebro.addstrategy(TestStrategy) # Create our store config = {'apiKey': apikey, 'secret': secret, 'enableRateLimit': True } # IMPORTANT NOTE - Kraken (and some other exchanges) will not return any values # for get cash or value if You have never held any LTC coins in your account. # So switch LTC to a coin you have funded previously if you get errors store = CCXTStore(exchange='binance', currency='USDT', config=config, retries=5, debug=False) # Get the broker and pass any kwargs if needed. # ---------------------------------------------- # Broker mappings have been added since some exchanges expect different values # to the defaults. Case in point, Kraken vs Bitmex. NOTE: Broker mappings are not # required if the broker uses the same values as the defaults in CCXTBroker. broker_mapping = { 'order_types': { bt.Order.Market: 'market', bt.Order.Limit: 'limit', bt.Order.Stop: 'stop-loss', #stop-loss for kraken, stop for bitmex bt.Order.StopLimit: 'stop limit' }, 'mappings':{ 'closed_order':{ 'key': 'status', 'value':'closed' }, 'canceled_order':{ 'key': 'result', 'value':1} } } broker = store.getbroker(broker_mapping=broker_mapping) cerebro.setbroker(broker) # Get our data # Drop newest will prevent us from loading partial data from incomplete candles hist_start_date = datetime.utcnow() - timedelta(minutes=50) data = store.getdata(dataname='ETH/USDT', name="ETHUSDT", timeframe=bt.TimeFrame.Minutes, fromdate=hist_start_date, compression=1, ohlcv_limit=50, drop_newest=True) #, historical=True) # Add the feed cerebro.adddata(data) cerebro.resampledata(data, timeframe=bt.TimeFrame.Minutes, compression=5, name="RESAMPLE") # Run the strategy cerebro.run()
which produced the following output:
2019-06-24 01:45:00 - RESAMPLE | Cash NA | O: 304.46 H: 306.21 L: 304.45 C: 306.04 V:1373.65208 SMA:305.352380952381 2019-06-24 09:48:15.019843 ETHUSDT Data Status: LIVE 2019-06-24 01:48:00 - ETHUSDT | Cash 97.0 | O: 305.5 H: 305.55 L: 305.11 C: 305.39 V:306.13727 SMA:305.31714285714287 2019-06-24 01:45:00 - RESAMPLE | Cash 97.0 | O: 304.46 H: 306.21 L: 304.45 C: 306.04 V:1373.65208 SMA:305.31714285714287 2019-06-24 01:49:00 - ETHUSDT | Cash 97.0 | O: 305.46 H: 306.46 L: 305.46 C: 306.46 V:215.59647 SMA:305.31904761904764 2019-06-24 01:45:00 - RESAMPLE | Cash 97.0 | O: 304.46 H: 306.21 L: 304.45 C: 306.04 V:1373.65208 SMA:305.31904761904764 Traceback (most recent call last): File "sample.py", line 107, in <module> cerebro.run() File "/Users/michael/.venv/bt-ccxt-store/lib/python3.6/site-packages/backtrader-1.9.74.123-py3.6.egg/backtrader/cerebro.py", line 1127, in run runstrat = self.runstrategies(iterstrat) File "/Users/michael/.venv/bt-ccxt-store/lib/python3.6/site-packages/backtrader-1.9.74.123-py3.6.egg/backtrader/cerebro.py", line 1298, in runstrategies self._runnext(runstrats) File "/Users/michael/.venv/bt-ccxt-store/lib/python3.6/site-packages/backtrader-1.9.74.123-py3.6.egg/backtrader/cerebro.py", line 1557, in _runnext dt0 = min((d for i, d in enumerate(dts) ValueError: min() arg is an empty sequence (bt-ccxt-store)
-
Here's the output of
remove fromdate
script and output:# !/usr/bin/env python # -*- coding: utf-8; py-indent-offset:4 -*- from __future__ import (absolute_import, division, print_function, unicode_literals) import sys import time from datetime import datetime, timedelta from datetime import datetime, timedelta import backtrader as bt import ccxt class TestStrategy(bt.Strategy): params = ( ('printlog', True), ) def log(self, txt, dt=None, doprint=False): ''' Logging function fot this strategy''' if self.params.printlog or doprint: dt = dt or bt.num2date(self.data.datetime[0]) print('%s, %s' % (dt, txt)) def start(self): self.counter = 0 print('START') def prenext(self): self.counter += 1 print('prenext len %d - counter %d' % (len(self), self.counter)) self.log("data#0: " + str(self.datas[0].datetime[0])) self.log("data#1: " + str(self.datas[1].datetime[0])) def __init__(self): self.macd = bt.indicators.MACDHisto(self.datas[0]) self.macd2 = bt.indicators.MACDHisto(self.datas[1]) def next(self): self.counter += 1 price_txt = "Counter: " + str(self.counter) + " Open/Close/High/Low/Volume: " + str(self.data0.open[0]) + " - "+ str(self.data0.close[0]) + " - " + str(self.data0.high[0]) + " - " + str(self.data0.low[0]) + " - " + str(self.data0.volume[0]) + " Len: "+ str(len(self.data0))# + " Time Frame:" + bt.TimeFrame.getname(self.data0._timeframe) + " Len: "+ str(len(self.data0)) self.log(price_txt) macd_txt = "MACD: {:.2f}, Histo: {:.2f}".format(self.macd.macd[0],self.macd.histo[0]) self.log("MACD#1: " + macd_txt) macd2_txt = "MACD: {:.2f}, Histo: {:.2f}".format(self.macd2.macd[0],self.macd2.histo[0]) self.log("MACD#2: " + macd2_txt) if __name__ == '__main__': cerebro = bt.Cerebro() #exchange = sys.argv[1] if len(sys.argv) > 1 else 'gdax' exchange = sys.argv[1] if len(sys.argv) > 1 else 'gateio' symbol = sys.argv[2] if len(sys.argv) > 2 else 'ETH/USDT' hist_start_date = datetime.utcnow() - timedelta(minutes=10) print('UTC NOW: ', datetime.utcnow()) print('hist_start_data: ', hist_start_date) print('Using symbol: ', symbol) # Create data feeds data_1m = bt.feeds.CCXT(exchange=exchange, symbol=symbol, name="1m", timeframe=bt.TimeFrame.Minutes, #fromdate=hist_start_date, compression=1) cerebro.adddata(data_1m) cerebro.replaydata(data_1m, timeframe=bt.TimeFrame.Minutes, name="1m", compression = 5) cerebro.addstrategy(TestStrategy) cerebro.run()
Output:
UTC NOW: 2019-06-24 02:30:45.980828 hist_start_data: 2019-06-24 02:20:45.980597 Using symbol: ETH/USDT START ... prenext len 163 - counter 163 2019-06-23 12:33:00, data#0: 737233.5229166667 2019-06-23 12:33:00, data#1: 737233.5229166667 dts[]: [737233.5236111111, 737233.5236111111] prenext len 164 - counter 164 2019-06-23 12:34:00, data#0: 737233.5236111111 2019-06-23 12:34:00, data#1: 737233.5236111111 dts[]: [737233.5243055555, 737233.5243055555] prenext len 165 - counter 165 2019-06-23 12:35:00, data#0: 737233.5243055555 2019-06-23 12:35:00, data#1: 737233.5243055555 dts[]: [737233.525, 737233.525] 2019-06-23 12:36:00, Counter: 166 Open/Close/High/Low/Volume: 312.57 - 312.45 - 312.71 - 312.45 - 99.16966564 Len: 166 2019-06-23 12:36:00, MACD#1: MACD: 0.07, Histo: -0.12 2019-06-23 12:36:00, MACD#2: MACD: 1.19, Histo: -0.09 dts[]: [737233.5256944444, 737233.5256944444] 2019-06-23 12:37:00, Counter: 167 Open/Close/High/Low/Volume: 312.45 - 312.54 - 312.54 - 312.36 - 21.3511 Len: 167 2019-06-23 12:37:00, MACD#1: MACD: 0.04, Histo: -0.12 2019-06-23 12:37:00, MACD#2: MACD: 1.19, Histo: -0.08 dts[]: [737233.526388889, 737233.526388889] 2019-06-23 12:38:00, Counter: 168 Open/Close/High/Low/Volume: 312.45 - 312.2 - 312.45 - 312.2 - 18.69802458 Len: 168 2019-06-23 12:38:00, MACD#1: MACD: -0.01, Histo: -0.13 2019-06-23 12:38:00, MACD#2: MACD: 1.17, Histo: -0.10 dts[]: [737233.5270833333, 737233.5270833333] 2019-06-23 12:39:00, Counter: 169 Open/Close/High/Low/Volume: 312.34 - 312.14 - 312.34 - 311.43 - 122.3391867563 Len: 169 2019-06-23 12:39:00, MACD#1: MACD: -0.05, Histo: -0.14 2019-06-23 12:39:00, MACD#2: MACD: 1.16, Histo: -0.11 dts[]: [737233.5277777778, 737233.5277777778] 2019-06-23 12:40:00, Counter: 170 Open/Close/High/Low/Volume: 311.44 - 311.79 - 311.79 - 311.43 - 22.0295 Len: 170 2019-06-23 12:40:00, MACD#1: MACD: -0.11, Histo: -0.16 2019-06-23 12:40:00, MACD#2: MACD: 1.13, Histo: -0.13 dts[]: [737233.5284722223, 737233.5284722223] ...
You see the
UTC NOW
? I don't know why the MACDs' date is at around2019-06-23 12:40:00
.
And I'm confused that:- using
fromdate
will case error - without
fromdate
, the script is still getting old data, but without error - And I don't know why it's getting old data from that time
How can these two happen in the same script?
I'm just lost.
- using