Help: Correlation between two symbols from IB

[aGiant]

Hi guys,

the codes from backtrader web for correlation between two symbols from IB didn't work, is this a bug?

class PearsonR(bt.ind.PeriodN):
    _mindatas = 2  # hint to the platform
    lines = ('correlation',)
    params = (('period', 20),)
    def next(self):
        #print(dir(self))
        print(self.dnames)
        #print(self.datas[0])
        c = scipy.stats.pearsonr(np.array(self.data0.get(size=self.p.period)),
                                 np.array(self.data1.get(size=self.p.period)))
        self.lines.correlation[0] = c[0]

cerebro = bt.Cerebro(stdstats=False)
store = bt.stores.IBStore(port=4002)#7499
data0 = store.getdata(dataname='EUR.USD-CASH-IDEALPRO', timeframe=bt.TimeFrame.Minutes)
cerebro.adddata(data0, name='d0')
cerebro.resampledata(data0, timeframe=bt.TimeFrame.Minutes, compression=1)

data1 = store.getdata(dataname='EUR.GBP-CASH-IDEALPRO', timeframe=bt.TimeFrame.Minutes)
    #cerebro.adddata(data1, name='d1')
cerebro.resampledata(data1, timeframe=bt.TimeFrame.Minutes, compression=1)
print(dir(cerebro))
print(cerebro.datasbyname)
print(cerebro.datas)
cerebro.addstrategy(St)

There was no self.data1 that should be transferred to this indicator. And there was no get() for self.data0 too if we printed dir(self.data0).

Many thanks! @backtrader

[aGiant]

Error:

 File "C:\ProgramData\Anaconda3\lib\site-packages\backtrader\cerebro.py", line 1127, in run
    runstrat = self.runstrategies(iterstrat)
  File "C:\ProgramData\Anaconda3\lib\site-packages\backtrader\cerebro.py", line 1264, in runstrategies
    strat._start()
  File "C:\ProgramData\Anaconda3\lib\site-packages\backtrader\strategy.py", line 385, in _start
    self._stage2()
  File "C:\ProgramData\Anaconda3\lib\site-packages\backtrader\lineiterator.py", line 181, in _stage2
    lineiterator._stage2()
  File "C:\ProgramData\Anaconda3\lib\site-packages\backtrader\lineiterator.py", line 177, in _stage2
    data._stage2()
  File "C:\ProgramData\Anaconda3\lib\site-packages\backtrader\lineiterator.py", line 181, in _stage2
    lineiterator._stage2()
  File "C:\ProgramData\Anaconda3\lib\site-packages\backtrader\lineiterator.py", line 177, in _stage2
    data._stage2()
  File "C:\ProgramData\Anaconda3\lib\site-packages\backtrader\lineiterator.py", line 181, in _stage2
    lineiterator._stage2()
  File "C:\ProgramData\Anaconda3\lib\site-packages\backtrader\lineiterator.py", line 177, in _stage2
    data._stage2()
  File "C:\ProgramData\Anaconda3\lib\site-packages\backtrader\lineiterator.py", line 181, in _stage2
    lineiterator._stage2()

[aGiant]

There is no get() function for self.data1:

b__', '__subclasshook__', '__truediv__', '__weakref__', '_add2stack', '_barstack', '_barstash', '_calendar', '_check', '_clone', '_compensate', '_compression', '_dataname', '_dlen', '_env', '_feed', '_ffilters', '_filters', '_fromstack', '_getline', '_getnexteos', '_getstatusname', '_gettz', '_gettzinput', '_id', '_last', '_laststatus', '_load', '_makeoperation', '_makeoperationown', '_minperiod', '_name', '_operation', '_operation_stage1', '_operation_stage2', '_operationown', '_operationown_stage1', '_operationown_stage2', '_opstage', '_owner', '_plotlabel', '_preloading', '_qcheck', '_roperation', '_save2stack', '_stage1', '_stage2', '_start', '_start_finish', '_started', '_store', '_tick_fill', '_tick_nullify', '_timeframe', '_timeoffset', '_tmoffset', '_tz', '_tzinput', '_updatebar', 'addfilter', 'addfilter_simple', 'addminperiod', 'advance', 'advance_peek', 'alias', 'aliased', 'array', 'backwards', 'clone', 'compensate', 'copyas', 'csv', 'data', 'date2num', 'do_qcheck', 'extend', 'forward', 'fromdate', 'frompackages', 'get_notifications', 'getenvironment', 'getfeed', 'getwriterheaders', 'getwriterinfo', 'getwritervalues', 'haslivedata', 'home', 'incminperiod', 'islive', 'l', 'line', 'line0', 'line1', 'line2', 'line3', 'line4', 'line5', 'line6', 'line_0', 'line_1', 'line_2', 'line_3', 'line_4', 'line_5', 'line_6', 'linealias', 'lines', 'load', 'minbuffer', 'next', 'nextstart', 'notifs', 'num2date', 'once', 'oncestart', 'p', 'packages', 'params', 'plotinfo', 'plotlabel', 'plotlines', 'preload', 'prenext', 'preonce', 'put_notification', 'qbuffer', 'replay', 'replaying', 'resample', 'resampling', 'reset', 'rewind', 'sessionend', 'sessionstart', 'setenvironment', 'setminperiod', 'start', 'stop', 'todate', 'updateminperiod']

[backtrader]

Sorry, but it is very difficult to follow the logic of what you are trying to show. There is no complete code sample, which makes it impossible to track any error.

For example:

@agiant said in Help: Correlation between two symbols from IB:

And there was no get() for self.data0 too if we printed dir(self.data0).

You have the wrong expectation that dir is going to tell you what's going to be executed. If you call get it will be there.

[aGiant]

@backtrader

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
import backtrader as bt
import redis
import itertools
import scipy.stats
import numpy as np

set_currency = ['EUR.USD-CASH-IDEALPRO', 'EUR.GBP-CASH-IDEALPRO', 'USD.GBP-CASH-IDEALPRO']
set_period_ind = ['MOM', 'RSI', 'STDDEV', 'LINEARREG_ANGLE', 'LINEARREG_SLOPE', 'SMA', 'CMO', 'ROCP']
set_period_ind_param = [3, 5, 8, 13, 15, 18, 23, 31, 44, 59, 80, 114, 169, 258, 402, 635]
set_state_ind_1 = ['HT_TRENDMODE'] #'HT_DCPERIOD', 'HT_DCPHASE',
set_state_ind_2 = ['HT_PHASOR'] #, 'HT_SINE'
set_oscillator_stoch = ['STOCH']
set_oscillator_stochrsi = ['STOCHRSI']
set_oscillator_macd = ['MACD']
set_oscillator_param = itertools.combinations(set_period_ind_param, 3)

r = redis.Redis(host='localhost', port=6379, db=0)
#r.set('foo','bar')
#r.set('dict_test',{"Name":"Pradeep", "Company":"SCTL", "Address":"Mumbai", "Location":"RCP"})

class PearsonR(bt.Indicator):
    #_mindatas = 2  # hint to the platform

    lines = ('correlation',)
    params = (('period', 20),)

    #def __init__(self):#, *args, **kwargs):
        #if 'datas' not in kwargs:
        #    raise bt.errors.BacktraderError()

        #self.addminperiod(self.p.period)

        #self.datasByName = kwargs['datas']

        #self.first = self.datasByName['first']
        #self.second = self.datasByName['second']
        #self.lines.correlation = scipy.stats.pearsonr
        #print(dir(self.datas))
        #print('*****************')
    def next(self):#, *args, **kwargs):
        #print(dir(self))
        print(self.dnames)
        #print(self.datas[0])
        c = scipy.stats.pearsonr(np.array(self.data0.get(size=self.p.period)),
                                 np.array(self.data1.get(size=self.p.period)))
        print(self.p.period,": ", c)
        #for i in np.array(self.data0.get(size=self.p.period)):
        #    print(i)
        print("-----------")
        #for i in np.array(self.data1.get(size=self.p.period)):
        #    print(i)
        exit()
        self.lines.correlation[0] = c[0]

class St(bt.Strategy):
    def __init__(self):
        print(self.dnames)
        self.indicators = {}
        self.indicators_names = []
        #mxL = max(set_period_ind_param)
        #L = len(self.data.open) - mxL - 1  # - max(set_pioder_ind_param)
        for s in set_period_ind:
            ret = getattr(bt.talib, s)
            for i, t in enumerate(set_period_ind_param):
                # print(s+'_'+str(t))
                self.indicators[s + '_' + str(t)] = ret(self.data0, timeperiod=t)
                # print(len(ret(data,timeperiod=t)))
        '''
        #oscilators
        set_oscillator_param = itertools.combinations(set_period_ind_param, 2)
        for s in set_oscillator_stoch:
            func = getattr(bt.talib, s)
            for i, item in enumerate(set_oscillator_param):
                # high, low, close and slow, fast1, type = 0, fast2, type = 0
                # d[s + '_fast_' + str(i)]
                #self.indicators[s + '_slowk_' + str(i)], self.indicators[s + '_slowd_' + str(i)]  = func(self.data.high, self.data.low, self.data.close, fastk_period=item[1], slowk_period=item[0], slowd_period=item[0])
                self.indicators[s + '_slowk_' + str(i)] = func(self.data.high, self.data.low, self.data.close, fastk_period=item[1], slowk_period=item[0], slowd_period=item[0])
                self.indicators[s + '_slowk_' + str(i)] = func(self.data.high, self.data.low, self.data.close, fastk_period=item[1], slowk_period=item[0], slowd_period=item[0])
                #MACD([input_arrays], [fastperiod=12], [slowperiod=26], [signalperiod=9])

        for s in set_oscillator_stoch:
            func = getattr(bt.talib, s)
            for i, item in enumerate(set_oscillator_param):
                # high, low, close and slow, fast1, type = 0, fast2, type = 0
                # d[s + '_fast_' + str(i)]
                #self.indicators[s + '_slowk_' + str(i)], self.indicators[s + '_slowd_' + str(i)]  = func(self.data.high, self.data.low, self.data.close, fastk_period=item[1], slowk_period=item[0], slowd_period=item[0])
                self.indicators[s + '_slowk_' + str(i)] = func(self.data.high, self.data.low, self.data.close, fastk_period=item[1], slowk_period=item[0], slowd_period=item[0])
                #MACD([input_arrays], [fastperiod=12], [slowperiod=26], [signalperiod=9])

        set_oscillator_param = itertools.combinations(set_period_ind_param, 3)
        for s in set_oscillator_macd:
            func = getattr(bt.talib, s)
            for i, item in enumerate(set_oscillator_param):
                # fastk, fastd = STOCHRSI(close, timeperiod=14, fastk_period=5, fastd_period=3, fastd_matype=0)
                # item = (14, 21, 34)
                self.indicators_names.append(s + '_' + str(i))
                self.indicators[s + '_' + str(i)] = func(self.data.close, fastperiod=item[1], slowperiod=item[2], signalperiod=item[0])
                #print(self.indicators[s + '_' + str(i)][0])
        '''
        #print(dir(self.data0))
        #print(dir(self.data1))
        #exit()
        #PearsonR(self.data0, self.data1)
        for i, t in enumerate(set_period_ind_param):
                # print(s+'_'+str(t))
                self.indicators['PearsonR' + '_' + str(t)] = bt.talib.CORREL(self.data0.close, self.data1.close, timeperiod=t)
        print(bt.talib.CORREL.__doc__)
        #print(dir(self))
        #print('{:.6f}'.format(self.data0.open[0]))
        #print('{:.6f}'.format(self.data1.open[0]))
        #print('{:.6f}'.format(self.data2.open[0]))
        #exit()
        #for i in self.indicators:
        #    print(self.indicators[i][0])
        #exit()
        #bt.indicators.Stochastic(self.data)
        #self.sma = getattr(bt.talib, "SMA")(self.data, period=20)#btind.SMA(self.data, period=20)
        #getattr(self, "sma") = getattr(bt.talib, "SMA")(self.data, period=20)#btind.SMA(self.data, period=20)
        #self.stoc = bt.talib.STOCH(self.data.high, self.data.low, self.data.close, fastk_period=14, slowk_period=3, slowd_period=3)


    def logdata(self):
        txt = []
        prices = {}
        txt.append('{}'.format(len(self)))
        key = '{}'.format(self.data.datetime.datetime(0).isoformat())
        txt.append(key)
        #print(self.data.Open)#(dir(self.data))
        open = self.data0.open[0]#(dir(self.data)
        txt.append(open)
        high = '{:.6f}'.format(self.data0.high[0])
        txt.append(high)
        low = '{:.6f}'.format(self.data0.low[0])
        txt.append(low)
        close = '{:.6f}'.format(self.data0.close[0])
        txt.append(close)
        volume = '{:.6f}'.format(self.data0.volume[0])
        txt.append(volume)
        #print(self.getindicators())
        #for i in self.indicators:
        #    print(i)

        #self.sma = bt.talib.SMA(self.data.open-self.data.close, period = 20)
        #print(bt.talib.STOCH.__doc__)
        #print(bt.talib.STOCHRSI.__doc__)
        #print(bt.talib.MACD.__doc__)
        #print(dir(sma2))
        #print(self.dnames)
        print('{:.6f}'.format(self.data0.open[0]))
        print('{:.6f}'.format(self.data1.open[0]))
        print('{:.6f}'.format(self.data2.open[0]))
        #print('{:.6f}'.format(self.data.open[0]))
        #print('{:.6f}'.format(self.data0_1[0]))
        #print('{:.6f}'.format(self.data1_1[0]))
        exit()
        #print(len(self.data.open))
        #print('{:.6f}'.format(self.sma.data.high[0]))
        #print(dir(sma2))0
        #print('{:.6f}'.format(self.sma[0]))
        #print(dir(bt.talib))
        #for i in self.indicators:
        #    print(self.indicators[i][0])
        #for i in self.indicators_names:
        #    print(len(self.indicators[i][0]))
        #exit()
        print(txt)

        #print(','.join(txt))
        #write data to redis should be here
        prices = {'Key':key, 'Open':open, 'High':high, 'Low':low, 'Close':close, 'Volume':volume}
        indicat = dict()
        for i in self.indicators:
            indicat[i] = self.indicators[i][0]
        prices.update(indicat)
        r.hmset(key, prices)

    def next(self):
        self.logdata()


def run(args=None):
    #TWTR
    cerebro = bt.Cerebro(stdstats=False)
    store = bt.stores.IBStore(port=4002)#7499
    #data0 = store.getdata(dataname='EUR.USD-CASH-IDEALPRO', timeframe=bt.TimeFrame.Minutes)
    #cerebro.adddata(data0, name='data0')
    #cerebro.resampledata(data0, timeframe=bt.TimeFrame.Minutes, compression=1)

    for c in set_currency:
        data1 = store.getdata(dataname=c, timeframe=bt.TimeFrame.Minutes)
    #cerebro.adddata(data1, name='data1')
        cerebro.resampledata(data1, timeframe=bt.TimeFrame.Minutes, compression=1)
    #print(dir(cerebro))
    #print(cerebro.datasbyname)
    #print(cerebro.datas)
    cerebro.addstrategy(St)

    cerebro.run()


if __name__ == '__main__':
    run()

The ideal is to calculate the indicators and save them in databse. Triangel correlations are considerred. But it didn't work at all.

[backtrader]

@agiant said in Help: Correlation between two symbols from IB:

But it didn't work at all.

Again: nobody can actually understand what didn't work or what worked. There is no error log, no log at all. The only thing you present is a bunch of code lines in which many are simply commented out, whether for a good reason or not, who knows. And many of those commented lines show things which are astonishing. For example:

@agiant said in Help: Correlation between two symbols from IB:

    #def __init__(self):#, *args, **kwargs):
        #if 'datas' not in kwargs:
        #    raise bt.errors.BacktraderError()

        #self.addminperiod(self.p.period)

        #self.datasByName = kwargs['datas']

Why are (or were) you expecting datas in kwargs?

Furthermore,

@agiant said in Help: Correlation between two symbols from IB:

    def next(self):#, *args, **kwargs):
        #print(dir(self))
        print(self.dnames)
        #print(self.datas[0])
        c = scipy.stats.pearsonr(np.array(self.data0.get(size=self.p.period)),
                                 np.array(self.data1.get(size=self.p.period)))
        print(self.p.period,": ", c)
        #for i in np.array(self.data0.get(size=self.p.period)):
        #    print(i)
        print("-----------")
        #for i in np.array(self.data1.get(size=self.p.period)):
        #    print(i)
        exit()

How is anything going to work if you call exit()?