For code/output blocks: Use ``` (aka backtick or grave accent) in a single line before and after the block. See: http://commonmark.org/help/

Unconverted data remains at .000 tried everything please help



  • Hey I've been trying to create a Binance csv backtester using this code;

    from future import (absolute_import, division, print_function, unicode_literals)
    import datetime
    import backtrader as bt
    import backtrader.feeds as btfeeds
    from strat.ex import TestStrategy
    import sys
    import os
    from dateutil import tz

    if name == 'main':
    cerebro = bt.Cerebro()
    cerebro.addstrategy(TestStrategy)
    modpath = os.path.dirname(os.path.abspath(sys.argv[0]))
    datapath = os.path.join(modpath, '/home/dev/PycharmProjects/Backtesting/csvdata/BTCUSDT-1m-data.csv')

    def parsedate(timestamp):
    if type(timestamp) is str: timestamp = int(timestamp) # put up with timestamp being in string format
    if timestamp > 1e10: timestamp // 1000 # put up with timestamp being in microseconds
    return datetime.datetime.utcfromtimestamp(float(timestamp)).replace(tzinfo=tz.tzutc())

    Create a Data Feed

    data = btfeeds.GenericCSVData(
    dataname=datapath,

    fromdate=datetime.datetime(2020, 6, 22),
    
    dtformat=('%Y-%m-%d %H:%M:%S'),
    
    nullvalue=0.0,
    
    datetime=0,
    high=2,
    low=3,
    open=1,
    close=4,
    volume=5,
    openinterest=-1,
    
    timeframe=bt.TimeFrame.Minutes,
    compression=30
    

    )

    cerebro.adddata(data)
    cerebro.broker.setcash(100000.0)
    cerebro.addsizer(bt.sizers.FixedSize, stake=0.05)
    cerebro.broker.setcommission(commission=0.005)

    print('Starting Balance: %.2f' % cerebro.broker.getvalue())

    cerebro.run()

    print('Final Balance: %.2f' % cerebro.broker.getvalue())

    This is the output I'm getting;
    Traceback (most recent call last):
    File "nancetester.py", line 53, in <module>
    cerebro.run()
    File "/home/dev/anaconda3/envs/Backtesting/lib/python3.7/site-packages/backtrader/cerebro.py", line 1127, in run
    runstrat = self.runstrategies(iterstrat)
    File "/home/dev/anaconda3/envs/Backtesting/lib/python3.7/site-packages/backtrader/cerebro.py", line 1212, in runstrategies
    data.preload()
    File "/home/dev/anaconda3/envs/Backtesting/lib/python3.7/site-packages/backtrader/feed.py", line 688, in preload
    while self.load():
    File "/home/dev/anaconda3/envs/Backtesting/lib/python3.7/site-packages/backtrader/feed.py", line 479, in load
    _loadret = self._load()
    File "/home/dev/anaconda3/envs/Backtesting/lib/python3.7/site-packages/backtrader/feed.py", line 710, in _load
    return self._loadline(linetokens)
    File "/home/dev/anaconda3/envs/Backtesting/lib/python3.7/site-packages/backtrader/feeds/csvgeneric.py", line 114, in _loadline
    dt = datetime.strptime(dtfield, dtformat)
    File "/home/dev/anaconda3/envs/Backtesting/lib/python3.7/_strptime.py", line 577, in _strptime_datetime
    tt, fraction, gmtoff_fraction = _strptime(data_string, format)
    File "/home/dev/anaconda3/envs/Backtesting/lib/python3.7/_strptime.py", line 362, in _strptime
    data_string[found.end():])
    ValueError: unconverted data remains: .000

    Please help, I've looked trough the forums for hours with no luck.

    Regards,

    cept0r



  • Could you include a short representation of your raw input data? Thanks. Please use the triple back quotes to make it more readible. See the top of the page for formatting instructions.

    For code/output blocks: Use ``` (aka backtick or grave accent) in a single line before and after the block. See: http://commonmark.org/help/
    


  • Will do, here is a screenshot of my csv data;
    link text



  • from __future__ import (absolute_import, division, print_function, unicode_literals)
    import datetime
    import backtrader as bt
    import backtrader.feeds as btfeeds
    from strat.ex import TestStrategy
    import sys
    import os
    from dateutil import tz
    
    if __name__ == '__main__':
        cerebro = bt.Cerebro()
        cerebro.addstrategy(TestStrategy)
        modpath = os.path.dirname(os.path.abspath(sys.argv[0]))
        datapath = os.path.join(modpath, '/home/dev/PycharmProjects/Backtesting/csvdata/BTCUSDT-1m-data.csv')
    
    
    def parsedate(timestamp):
        if type(timestamp) is str: timestamp = int(timestamp)  # put up with timestamp being in string format
        if timestamp > 1e10: timestamp // 1000  # put up with timestamp being in microseconds
        return datetime.datetime.utcfromtimestamp(float(timestamp)).replace(tzinfo=tz.tzutc())
    
    
    # Create a Data Feed
    data = btfeeds.GenericCSVData(
        dataname=datapath,
    
        fromdate=datetime.datetime(2020, 6, 22),
    
        dtformat=('%Y-%m-%d %H:%M:%S'),
    
        nullvalue=0.0,
    
        datetime=0,
        high=2,
        low=3,
        open=1,
        close=4,
        volume=5,
        openinterest=-1,
    
        timeframe=bt.TimeFrame.Minutes,
        compression=30
    )
    
    cerebro.adddata(data)
    cerebro.broker.setcash(100000.0)
    cerebro.addsizer(bt.sizers.FixedSize, stake=0.05)
    cerebro.broker.setcommission(commission=0.005)
    
    print('Starting Balance: %.2f' % cerebro.broker.getvalue())
    
    cerebro.run()
    
    print('Final Balance: %.2f' % cerebro.broker.getvalue())
    
    cerebro.plot()
    


  • #  Output:
    Traceback (most recent call last):
    File "nancetester.py", line 53, in <module>
    cerebro.run()
    File "/home/dev/anaconda3/envs/Backtesting/lib/python3.7/site-packages/backtrader/cerebro.py", line 1127, in run
    runstrat = self.runstrategies(iterstrat)
    File "/home/dev/anaconda3/envs/Backtesting/lib/python3.7/site-packages/backtrader/cerebro.py", line 1212, in runstrategies
    data.preload()
    File "/home/dev/anaconda3/envs/Backtesting/lib/python3.7/site-packages/backtrader/feed.py", line 688, in preload
    while self.load():
    File "/home/dev/anaconda3/envs/Backtesting/lib/python3.7/site-packages/backtrader/feed.py", line 479, in load
    _loadret = self._load()
    File "/home/dev/anaconda3/envs/Backtesting/lib/python3.7/site-packages/backtrader/feed.py", line 710, in _load
    return self._loadline(linetokens)
    File "/home/dev/anaconda3/envs/Backtesting/lib/python3.7/site-packages/backtrader/feeds/csvgeneric.py", line 114, in _loadline
    dt = datetime.strptime(dtfield, dtformat)
    File "/home/dev/anaconda3/envs/Backtesting/lib/python3.7/_strptime.py", line 577, in _strptime_datetime
    tt, fraction, gmtoff_fraction = _strptime(data_string, format)
    File "/home/dev/anaconda3/envs/Backtesting/lib/python3.7/_strptime.py", line 362, in _strptime
    data_string[found.end():])
    ValueError: unconverted data remains: .000
    


  • I solved it! instead of using

    GenericCSVData
    

    I used this instead;

    btc = pd.read_csv('/home/dev/PycharmProjects/Backtesting/csvdata/BTCUSDT-1m-data.csv',
                      index_col='timestamp',
                      parse_dates=True)
    
    
    feed = bt.feeds.PandasData(
        dataname=btc,
    
        fromdate=datetime.datetime(2020, 1, 1),
        todate=datetime.datetime(2020, 1, 2),
    
        timeframe=bt.TimeFrame.Minutes,
    
    

    Now it works! Sorry for wasting anybody's time.



  • @cept0r It's never a waste to ask as it lead you to thinking through the solution. Nice!



  • Hello, I encounter the same error as cept0r while using the 30m data from binance. Because the timestamp retrieve from Binance shows 2017-08-17 04:00:00.000 . You can see that there's additional .000 on the right.

    Is there a way to feed the time while omitting .000 in backtrader?

    Thank you.



  • @kian-hong-Tan said in Unconverted data remains at .000 tried everything please help:

    2017-08-17 04:00:00.000

    Add in a format string. Backtrader takes this format string and passes it directly to datetime.datetime.strptime. So the string you are looking for needs to be for strptime. Have a look at this stackoverflow question. I haven't tested this but try the following.

    format='%Y-%m-%d %H:%M:%S.%f'
    

    as in:

    feed = bt.feeds.PandasData(
        dataname=btc,
    
        fromdate=datetime.datetime(2020, 1, 1),
        todate=datetime.datetime(2020, 1, 2),
        format='%Y-%m-%d %H:%M:%S.%f'
    
        timeframe=bt.TimeFrame.Minutes,
    




  • Tks. This is my raw data. I do not want to feed the .000 into backtrader actually.

    001b2429-9dc4-4c24-88e9-01a0ec552746-image.png

    This is my code for data feed from csv:

    if __name__ == '__main__':
    cerebro = bt.Cerebro()
    cerebro.addstrategy(TestStrategy)
    
    data = bt.feeds.GenericCSVData(
    dataname='BTCUSDT-30m-data.csv',
    
    fromdate=datetime.datetime(2018, 1, 1),
    todate=datetime.datetime(2020, 8, 8),
        
    nullvalue=0.0,                            
    
    timeframe=bt.TimeFrame.Minutes,            
    compression=1,                            
    dtformat=('%Y-%m-%d %H:%M:%S'),  
    datetime=0,
    time=-1,
    high=2,
    low=3,
    open=1,
    close=4,
    volume=5,
    openinterest=-1
    )


  • @kian-hong-Tan said in Unconverted data remains at .000 tried everything please help:

    I do not want to feed the .000 into backtrader actually.

    There are two ways:

    • write new data feed which will skip .000 while processing lines of the csv file
    • pre-process data feeds and remove .000 from the data feed files, then use standard bt csv data feed


  • @ab_trader Ya, i remove .000 before feeding into bt. T q so much.


Log in to reply
 

});