For code/output blocks: Use ``` (aka backtick or grave accent) in a single line before and after the block. See: http://commonmark.org/help/

Resample live feed from CCXT problem



  • Guys,

    I'm trying to get macd of 1minute and 5minute timeframe.
    But I'm getting ValueError: min() arg is an empty sequence error.

    Here's the script:

    # !/usr/bin/env python
    # -*- coding: utf-8; py-indent-offset:4 -*-
    from __future__ import (absolute_import, division, print_function,
                            unicode_literals)
    import sys
    import time
    from datetime import datetime, timedelta
    from datetime import datetime, timedelta
    
    import backtrader as bt
    import ccxt
    
    class TestStrategy(bt.Strategy):
        params = (
            ('printlog', True),
        )
    
        def log(self, txt, dt=None, doprint=False):
            ''' Logging function fot this strategy'''
            if self.params.printlog or doprint:
                dt = dt or bt.num2date(self.data.datetime[0])
                print('%s, %s' % (dt, txt))
    
        def start(self):
            self.counter = 0
            print('START')
    
        def prenext(self):
            self.counter += 1
            print('prenext len %d - counter %d' % (len(self), self.counter))
            self.log("data#0: " + str(self.datas[0].datetime[0]))
            self.log("data#1: " + str(self.datas[1].datetime[0]))
    
        def __init__(self):
            self.macd = bt.indicators.MACDHisto(self.datas[0])
            self.macd2 = bt.indicators.MACDHisto(self.datas[1])
    
    
        def next(self):
            self.counter += 1
            price_txt = "Counter: " + str(self.counter) + " Open/Close/High/Low/Volume: " + str(self.data0.open[0]) + " - "+ str(self.data0.close[0]) + " - " + str(self.data0.high[0]) + " - " + str(self.data0.low[0])  + " - " + str(self.data0.volume[0]) + " Len: "+ str(len(self.data0))# + " Time Frame:" + bt.TimeFrame.getname(self.data0._timeframe) + " Len: "+ str(len(self.data0))
            self.log(price_txt)
            macd_txt = "MACD: {:.2f}, Histo: {:.2f}".format(self.macd.macd[0],self.macd.histo[0])
            self.log("MACD#1: " + macd_txt)
            macd2_txt = "MACD: {:.2f}, Histo: {:.2f}".format(self.macd2.macd[0],self.macd2.histo[0])
            self.log("MACD#2: " + macd2_txt)
    
    if __name__ == '__main__':
        cerebro = bt.Cerebro()
    
        #exchange = sys.argv[1] if len(sys.argv) > 1 else 'gdax'
        exchange = sys.argv[1] if len(sys.argv) > 1 else 'okex'
        symbol = sys.argv[2] if len(sys.argv) > 2 else 'ETH/USDT'
    
        hist_start_date = datetime.utcnow() - timedelta(minutes=10)
        print('UTC NOW: ', datetime.utcnow())
        print('hist_start_data: ', hist_start_date)
        print('Using symbol: ', symbol)
    
        # Create data feeds
        data_1m = bt.feeds.CCXT(exchange=exchange, symbol=symbol, name="1m",
                                   timeframe=bt.TimeFrame.Minutes, fromdate=hist_start_date,compression=1)
        cerebro.adddata(data_1m)
    
        cerebro.replaydata(data_1m, timeframe=bt.TimeFrame.Minutes, name="2m",compression=2)
        
        
        cerebro.addstrategy(TestStrategy)
        cerebro.run()
    

    And the output:

    $ python stage1.py
    UTC NOW:  2019-06-21 08:33:35.360573
    hist_start_data:  2019-06-21 08:23:35.360331
    Using symbol:  ETH/USDT
    START
    dts[]:  [737231.35, 737231.35]
    prenext len 1 - counter 1
    2019-06-21 08:24:00, data#0: 737231.35
    2019-06-21 08:24:00, data#1: 737231.35
    dts[]:  [737231.3506944445, None]
    prenext len 2 - counter 2
    2019-06-21 08:25:00, data#0: 737231.3506944445
    2019-06-21 08:25:00, data#1: 737231.35
    dts[]:  [737231.3513888889, 737231.3513888889]
    prenext len 3 - counter 3
    2019-06-21 08:26:00, data#0: 737231.3513888889
    2019-06-21 08:26:00, data#1: 737231.3513888889
    dts[]:  [737231.3520833333, None]
    prenext len 4 - counter 4
    2019-06-21 08:27:00, data#0: 737231.3520833333
    2019-06-21 08:27:00, data#1: 737231.3513888889
    dts[]:  [737231.3527777778, 737231.3527777778]
    prenext len 5 - counter 5
    2019-06-21 08:28:00, data#0: 737231.3527777778
    2019-06-21 08:28:00, data#1: 737231.3527777778
    dts[]:  [737231.3534722222, None]
    prenext len 6 - counter 6
    2019-06-21 08:29:00, data#0: 737231.3534722222
    2019-06-21 08:29:00, data#1: 737231.3527777778
    dts[]:  [737231.3541666666, 737231.3541666666]
    prenext len 7 - counter 7
    2019-06-21 08:30:00, data#0: 737231.3541666666
    2019-06-21 08:30:00, data#1: 737231.3541666666
    dts[]:  [737231.3548611111, None]
    prenext len 8 - counter 8
    2019-06-21 08:31:00, data#0: 737231.3548611111
    2019-06-21 08:31:00, data#1: 737231.3541666666
    dts[]:  [737231.3555555556, 737231.3555555556]
    prenext len 9 - counter 9
    2019-06-21 08:32:00, data#0: 737231.3555555556
    2019-06-21 08:32:00, data#1: 737231.3555555556
    dts[]:  [737231.35625, None]
    prenext len 10 - counter 10
    2019-06-21 08:33:00, data#0: 737231.35625
    2019-06-21 08:33:00, data#1: 737231.3555555556
    dts[]:  [None, 737231.3569444445]
    Traceback (most recent call last):
      File "stage1.py", line 69, in <module>
        cerebro.run()
      File "/Users/michael/.venv/backtrader-ccxt/lib/python3.6/site-packages/backtrader/cerebro.py", line 1127, in run
        runstrat = self.runstrategies(iterstrat)
      File "/Users/michael/.venv/backtrader-ccxt/lib/python3.6/site-packages/backtrader/cerebro.py", line 1298, in runstrategies
        self._runnext(runstrats)
      File "/Users/michael/.venv/backtrader-ccxt/lib/python3.6/site-packages/backtrader/cerebro.py", line 1558, in _runnext
        dt0 = min((d for i, d in enumerate(dts)
    ValueError: min() arg is an empty sequence
    

    You may not see the below:

    2019-06-21 08:32:00, data#0: 737231.3555555556
    2019-06-21 08:32:00, data#1: 737231.3555555556
    dts[]:  [737231.35625, None]
    

    because I changed the source code to print them out.

    Any ideas?

    What am I doing wrong?

    Any help appreciated!


  • administrators

    This seems related to your usage of the CCXT fork, as per your other message:

    The fork will have to handle it.



  • @backtrader Hi Backtrader, I guess not.
    Because I tried bt-ccxt-store too. It's the same error.

    And it's going well if remove the fromdate. And also, leads to a before time point, don't know why.

    Here's goes the bt-ccxt-store version script, which is almost the same:

    
    # !/usr/bin/env python
    # -*- coding: utf-8; py-indent-offset:4 -*-
    from __future__ import (absolute_import, division, print_function,
                            unicode_literals)
    import sys
    import time
    from datetime import datetime, timedelta
    
    import backtrader as bt
    from ccxtbt import CCXTFeed
    
    class TestStrategy(bt.Strategy):
        params = (
            ('printlog', True),
        )
    
        def log(self, txt, dt=None, doprint=False):
            ''' Logging function fot this strategy'''
            if self.params.printlog or doprint:
                dt = dt or bt.num2date(self.data.datetime[0])
                print('%s, %s' % (dt, txt))
    
        def start(self):
            self.counter = 0
            print('START')
    
        def prenext(self):
            self.counter += 1
            print('prenext len %d - counter %d' % (len(self), self.counter))
    
        def __init__(self):
            self.macd = bt.indicators.MACDHisto(self.datas[0])
            self.macd2 = bt.indicators.MACDHisto(self.datas[1])
    
    
        def next(self):
            self.counter += 1
            price_txt = "Counter: " + str(self.counter) + " Open/Close/High/Low/Volume: " + str(self.data0.open[0]) + " - "+ str(self.data0.close[0]) + " - " + str(self.data0.high[0]) + " - " + str(self.data0.low[0])  + " - " + str(self.data0.volume[0]) + " Len: "+ str(len(self.data0))# + " Time Frame:" + bt.TimeFrame.getname(self.data0._timeframe) + " Len: "+ str(len(self.data0))
            self.log(price_txt)
            macd_txt = "MACD: {:.2f}, Histo: {:.2f}".format(self.macd.macd[0],self.macd.histo[0])
            self.log("MACD#1: " + macd_txt)
            macd2_txt = "MACD: {:.2f}, Histo: {:.2f}".format(self.macd2.macd[0],self.macd2.histo[0])
            self.log("MACD#2: " + macd2_txt)
        
        def notify_data(self, data, status, *args, **kwargs):
            dn = data._name
            dt = datetime.now()
            msg= 'Data Status: {}'.format(data._getstatusname(status))
            print(dt,dn,msg)
            if data._getstatusname(status) == 'LIVE':
                self.live_data = True
            else:
                self.live_data = False
    
    if __name__ == '__main__':
        #cerebro = bt.Cerebro(quicknotify=True)
        cerebro = bt.Cerebro()
    
        
    
        #exchange = sys.argv[1] if len(sys.argv) > 1 else 'gdax'
        exchange = sys.argv[1] if len(sys.argv) > 1 else 'binance'
        symbol = sys.argv[2] if len(sys.argv) > 2 else 'ETH/USDT'
    
        #store = CCXTStore(exchange=exchange, currency='LTC', config=config, retries=5, debug=False)
    
        hist_start_date = datetime.utcnow() - timedelta(minutes=10)
        print('UTC NOW: ', datetime.utcnow())
        print('hist_start_data: ', hist_start_date)
        print('Using symbol: ', symbol)
    
        data = CCXTFeed(exchange=exchange,
                                 dataname=symbol,
                                 timeframe=bt.TimeFrame.Minutes,
                                 #fromdate=hist_start_date,
                                 #todate=datetime(2019, 1, 1, 0, 2),
                                 compression=1,
                                 ohlcv_limit=2,
                                 currency='USDT',
                                 retries=5,
                                 config={'enableRateLimit': True, 'nonce': lambda: str(int(time.time() * 1000))})
    
        cerebro.adddata(data)
        cerebro.resampledata(data, timeframe=bt.TimeFrame.Minutes, compression=5)
    
        cerebro.addstrategy(TestStrategy)
        # Run the strategy
        cerebro.run()
    
    

    With the following output:

    prenext len 47 - counter 47
    prenext len 48 - counter 48
    prenext len 49 - counter 49
    Traceback (most recent call last):
      File "ccxtbttest.py", line 88, in <module>
        cerebro.adddata(data)
      File "/Users/michael/.venv/bt-ccxt-store/lib/python3.6/site-packages/backtrader-1.9.74.123-py3.6.egg/backtrader/cerebro.py", line 1127, in run
        runstrat = self.runstrategies(iterstrat)
      File "/Users/michael/.venv/bt-ccxt-store/lib/python3.6/site-packages/backtrader-1.9.74.123-py3.6.egg/backtrader/cerebro.py", line 1298, in runstrategies
        self._runnext(runstrats)
      File "/Users/michael/.venv/bt-ccxt-store/lib/python3.6/site-packages/backtrader-1.9.74.123-py3.6.egg/backtrader/cerebro.py", line 1557, in _runnext
        dt0 = min((d for i, d in enumerate(dts)
    ValueError: min() arg is an empty sequence
    (bt-ccxt-store) 
    
    

    And also Another script:

       
    from ccxtbt import CCXTStore
    import backtrader as bt
    from datetime import datetime, timedelta
    
    
    class TestStrategy(bt.Strategy):
    
        def __init__(self):
    
            self.sma = bt.indicators.SMA(self.datas[0],period=21)
            #self.macd5 = bt.indicators.MACD(self.datas[1])
    
        def next(self):
    
            # Get cash and balance
            # New broker method that will let you get the cash and balance for
            # any wallet. It also means we can disable the getcash() and getvalue()
            # rest calls before and after next which slows things down.
    
            # NOTE: If you try to get the wallet balance from a wallet you have
            # never funded, a KeyError will be raised! Change LTC below as approriate
            if self.live_data:
                cash, value = self.broker.get_wallet_balance('USDT')
            else:
                # Avoid checking the balance during a backfill. Otherwise, it will
                # Slow things down.
                cash = 'NA'
    
            for data in self.datas:
    
                print('{} - {} \t| Cash {} | O: {} H: {} L: {} C: {} V:{} \tSMA:{}'.format(data.datetime.datetime(),
                    data._name, cash, data.open[0], data.high[0], data.low[0], data.close[0], data.volume[0],
                    self.sma[0]))
    
        def notify_data(self, data, status, *args, **kwargs):
            dn = data._name
            dt = datetime.now()
            msg= 'Data Status: {}'.format(data._getstatusname(status))
            print(dt,dn,msg)
            if data._getstatusname(status) == 'LIVE':
                self.live_data = True
            else:
                self.live_data = False
    
    
    
    apikey = 'x'
    secret = 'xx'
    
    cerebro = bt.Cerebro(quicknotify=True)
    
    
    # Add the strategy
    cerebro.addstrategy(TestStrategy)
    
    # Create our store
    config = {'apiKey': apikey,
                'secret': secret,
                'enableRateLimit': True
                }
    
    # IMPORTANT NOTE - Kraken (and some other exchanges) will not return any values
    # for get cash or value if You have never held any LTC coins in your account.
    # So switch LTC to a coin you have funded previously if you get errors
    store = CCXTStore(exchange='binance', currency='USDT', config=config, retries=5, debug=False)
    
    
    # Get the broker and pass any kwargs if needed.
    # ----------------------------------------------
    # Broker mappings have been added since some exchanges expect different values
    # to the defaults. Case in point, Kraken vs Bitmex. NOTE: Broker mappings are not
    # required if the broker uses the same values as the defaults in CCXTBroker.
    broker_mapping = {
        'order_types': {
            bt.Order.Market: 'market',
            bt.Order.Limit: 'limit',
            bt.Order.Stop: 'stop-loss', #stop-loss for kraken, stop for bitmex
            bt.Order.StopLimit: 'stop limit'
        },
        'mappings':{
            'closed_order':{
                'key': 'status',
                'value':'closed'
                },
            'canceled_order':{
                'key': 'result',
                'value':1}
                }
        }
    
    broker = store.getbroker(broker_mapping=broker_mapping)
    cerebro.setbroker(broker)
    
    # Get our data
    # Drop newest will prevent us from loading partial data from incomplete candles
    hist_start_date = datetime.utcnow() - timedelta(minutes=50)
    data = store.getdata(dataname='ETH/USDT', name="ETHUSDT",
                             timeframe=bt.TimeFrame.Minutes, fromdate=hist_start_date,
                             compression=1, ohlcv_limit=50, drop_newest=True) #, historical=True)
    
    # Add the feed
    cerebro.adddata(data)
    cerebro.resampledata(data, timeframe=bt.TimeFrame.Minutes, compression=5, name="RESAMPLE")
    
    # Run the strategy
    cerebro.run()
    
    
    

    which produced the following output:

    2019-06-24 01:45:00 - RESAMPLE 	| Cash NA | O: 304.46 H: 306.21 L: 304.45 C: 306.04 V:1373.65208 	SMA:305.352380952381
    2019-06-24 09:48:15.019843 ETHUSDT Data Status: LIVE
    2019-06-24 01:48:00 - ETHUSDT 	| Cash 97.0 | O: 305.5 H: 305.55 L: 305.11 C: 305.39 V:306.13727 	SMA:305.31714285714287
    2019-06-24 01:45:00 - RESAMPLE 	| Cash 97.0 | O: 304.46 H: 306.21 L: 304.45 C: 306.04 V:1373.65208 	SMA:305.31714285714287
    2019-06-24 01:49:00 - ETHUSDT 	| Cash 97.0 | O: 305.46 H: 306.46 L: 305.46 C: 306.46 V:215.59647 	SMA:305.31904761904764
    2019-06-24 01:45:00 - RESAMPLE 	| Cash 97.0 | O: 304.46 H: 306.21 L: 304.45 C: 306.04 V:1373.65208 	SMA:305.31904761904764
    Traceback (most recent call last):
      File "sample.py", line 107, in <module>
        cerebro.run()
      File "/Users/michael/.venv/bt-ccxt-store/lib/python3.6/site-packages/backtrader-1.9.74.123-py3.6.egg/backtrader/cerebro.py", line 1127, in run
        runstrat = self.runstrategies(iterstrat)
      File "/Users/michael/.venv/bt-ccxt-store/lib/python3.6/site-packages/backtrader-1.9.74.123-py3.6.egg/backtrader/cerebro.py", line 1298, in runstrategies
        self._runnext(runstrats)
      File "/Users/michael/.venv/bt-ccxt-store/lib/python3.6/site-packages/backtrader-1.9.74.123-py3.6.egg/backtrader/cerebro.py", line 1557, in _runnext
        dt0 = min((d for i, d in enumerate(dts)
    ValueError: min() arg is an empty sequence
    (bt-ccxt-store) 
    


  • @backtrader

    Here's the output of remove fromdate script and output:

    # !/usr/bin/env python
    # -*- coding: utf-8; py-indent-offset:4 -*-
    from __future__ import (absolute_import, division, print_function,
                            unicode_literals)
    import sys
    import time
    from datetime import datetime, timedelta
    from datetime import datetime, timedelta
    
    import backtrader as bt
    import ccxt
    
    class TestStrategy(bt.Strategy):
        params = (
            ('printlog', True),
        )
    
        def log(self, txt, dt=None, doprint=False):
            ''' Logging function fot this strategy'''
            if self.params.printlog or doprint:
                dt = dt or bt.num2date(self.data.datetime[0])
                print('%s, %s' % (dt, txt))
    
        def start(self):
            self.counter = 0
            print('START')
    
        def prenext(self):
            self.counter += 1
            print('prenext len %d - counter %d' % (len(self), self.counter))
            self.log("data#0: " + str(self.datas[0].datetime[0]))
            self.log("data#1: " + str(self.datas[1].datetime[0]))
    
        def __init__(self):
            self.macd = bt.indicators.MACDHisto(self.datas[0])
            self.macd2 = bt.indicators.MACDHisto(self.datas[1])
    
    
        def next(self):
            self.counter += 1
            price_txt = "Counter: " + str(self.counter) + " Open/Close/High/Low/Volume: " + str(self.data0.open[0]) + " - "+ str(self.data0.close[0]) + " - " + str(self.data0.high[0]) + " - " + str(self.data0.low[0])  + " - " + str(self.data0.volume[0]) + " Len: "+ str(len(self.data0))# + " Time Frame:" + bt.TimeFrame.getname(self.data0._timeframe) + " Len: "+ str(len(self.data0))
            self.log(price_txt)
            macd_txt = "MACD: {:.2f}, Histo: {:.2f}".format(self.macd.macd[0],self.macd.histo[0])
            self.log("MACD#1: " + macd_txt)
            macd2_txt = "MACD: {:.2f}, Histo: {:.2f}".format(self.macd2.macd[0],self.macd2.histo[0])
            self.log("MACD#2: " + macd2_txt)
    
    if __name__ == '__main__':
        cerebro = bt.Cerebro()
    
        #exchange = sys.argv[1] if len(sys.argv) > 1 else 'gdax'
        exchange = sys.argv[1] if len(sys.argv) > 1 else 'gateio'
        symbol = sys.argv[2] if len(sys.argv) > 2 else 'ETH/USDT'
    
        hist_start_date = datetime.utcnow() - timedelta(minutes=10)
        print('UTC NOW: ', datetime.utcnow())
        print('hist_start_data: ', hist_start_date)
        print('Using symbol: ', symbol)
    
        # Create data feeds
        data_1m = bt.feeds.CCXT(exchange=exchange, symbol=symbol, name="1m",
                                   timeframe=bt.TimeFrame.Minutes, #fromdate=hist_start_date,
                                   compression=1)
        cerebro.adddata(data_1m)
    
        cerebro.replaydata(data_1m, timeframe=bt.TimeFrame.Minutes, name="1m", compression = 5)
        
        
        cerebro.addstrategy(TestStrategy)
        cerebro.run()
    

    Output:

    UTC NOW:  2019-06-24 02:30:45.980828
    hist_start_data:  2019-06-24 02:20:45.980597
    Using symbol:  ETH/USDT
    START
    
    ...
    prenext len 163 - counter 163
    2019-06-23 12:33:00, data#0: 737233.5229166667
    2019-06-23 12:33:00, data#1: 737233.5229166667
    dts[]:  [737233.5236111111, 737233.5236111111]
    prenext len 164 - counter 164
    2019-06-23 12:34:00, data#0: 737233.5236111111
    2019-06-23 12:34:00, data#1: 737233.5236111111
    dts[]:  [737233.5243055555, 737233.5243055555]
    prenext len 165 - counter 165
    2019-06-23 12:35:00, data#0: 737233.5243055555
    2019-06-23 12:35:00, data#1: 737233.5243055555
    dts[]:  [737233.525, 737233.525]
    2019-06-23 12:36:00, Counter: 166 Open/Close/High/Low/Volume: 312.57 - 312.45 - 312.71 - 312.45 - 99.16966564 Len: 166
    2019-06-23 12:36:00, MACD#1: MACD: 0.07, Histo: -0.12
    2019-06-23 12:36:00, MACD#2: MACD: 1.19, Histo: -0.09
    dts[]:  [737233.5256944444, 737233.5256944444]
    2019-06-23 12:37:00, Counter: 167 Open/Close/High/Low/Volume: 312.45 - 312.54 - 312.54 - 312.36 - 21.3511 Len: 167
    2019-06-23 12:37:00, MACD#1: MACD: 0.04, Histo: -0.12
    2019-06-23 12:37:00, MACD#2: MACD: 1.19, Histo: -0.08
    dts[]:  [737233.526388889, 737233.526388889]
    2019-06-23 12:38:00, Counter: 168 Open/Close/High/Low/Volume: 312.45 - 312.2 - 312.45 - 312.2 - 18.69802458 Len: 168
    2019-06-23 12:38:00, MACD#1: MACD: -0.01, Histo: -0.13
    2019-06-23 12:38:00, MACD#2: MACD: 1.17, Histo: -0.10
    dts[]:  [737233.5270833333, 737233.5270833333]
    2019-06-23 12:39:00, Counter: 169 Open/Close/High/Low/Volume: 312.34 - 312.14 - 312.34 - 311.43 - 122.3391867563 Len: 169
    2019-06-23 12:39:00, MACD#1: MACD: -0.05, Histo: -0.14
    2019-06-23 12:39:00, MACD#2: MACD: 1.16, Histo: -0.11
    dts[]:  [737233.5277777778, 737233.5277777778]
    2019-06-23 12:40:00, Counter: 170 Open/Close/High/Low/Volume: 311.44 - 311.79 - 311.79 - 311.43 - 22.0295 Len: 170
    2019-06-23 12:40:00, MACD#1: MACD: -0.11, Histo: -0.16
    2019-06-23 12:40:00, MACD#2: MACD: 1.13, Histo: -0.13
    dts[]:  [737233.5284722223, 737233.5284722223]
    ...
    

    You see the UTC NOW? I don't know why the MACDs' date is at around 2019-06-23 12:40:00.
    And I'm confused that:

    1. using fromdate will case error
    2. without fromdate, the script is still getting old data, but without error
    3. And I don't know why it's getting old data from that time

    How can these two happen in the same script?

    I'm just lost.


Log in to reply
 

});