@backtrader Right. I don't see how I can use PyFolio with BackTrader in that case then, or how others in my situation can either, without downgrading each package as and when a respective error appears.
I wonder if anyone has a definitive list of the dependencies and the versions needed for it to work correctly?
@ab_trader Ah yes, I see, cheers. And the errors? They seem to be causing many of the graphs to be plotting incorrectly/incompletely.
Also, my output of the worst drawdown periods seems to be incorrect in a way that's not caused by the randomness of the sampling. For example, considering the data sources are the same, when you compare my output to this:
I am trying to replicate the sample shown on https://www.backtrader.com/docu/analyzers/pyfolio.html on my computer so that I can begin to use it on my own strategy knowing it works properly. I am using python 3.6 and have downgraded to the version of PyFolio used for the backtrader integration (0.5.1).
The code I have is identical to the sample in the link:
from __future__ import (absolute_import, division, print_function,
unicode_literals)
import argparse
import datetime
import random
import backtrader as bt
class St(bt.Strategy):
params = (
('printout', False),
('stake', 1000),
)
def __init__(self):
pass
def start(self):
if self.p.printout:
txtfields = list()
txtfields.append('Len')
txtfields.append('Datetime')
txtfields.append('Open')
txtfields.append('High')
txtfields.append('Low')
txtfields.append('Close')
txtfields.append('Volume')
txtfields.append('OpenInterest')
print(','.join(txtfields))
def next(self):
if self.p.printout:
# Print only 1st data ... is just a check that things are running
txtfields = list()
txtfields.append('%04d' % len(self))
txtfields.append(self.data.datetime.datetime(0).isoformat())
txtfields.append('%.2f' % self.data0.open[0])
txtfields.append('%.2f' % self.data0.high[0])
txtfields.append('%.2f' % self.data0.low[0])
txtfields.append('%.2f' % self.data0.close[0])
txtfields.append('%.2f' % self.data0.volume[0])
txtfields.append('%.2f' % self.data0.openinterest[0])
print(','.join(txtfields))
# Data 0
for data in self.datas:
toss = random.randint(1, 10)
curpos = self.getposition(data)
if curpos.size:
if toss > 5:
size = curpos.size // 2
self.sell(data=data, size=size)
if self.p.printout:
print('SELL {} @%{}'.format(size, data.close[0]))
elif toss < 5:
self.buy(data=data, size=self.p.stake)
if self.p.printout:
print('BUY {} @%{}'.format(self.p.stake, data.close[0]))
def runstrat(args=None):
args = parse_args(args)
cerebro = bt.Cerebro()
cerebro.broker.set_cash(args.cash)
dkwargs = dict()
if args.fromdate:
fromdate = datetime.datetime.strptime(args.fromdate, '%Y-%m-%d')
dkwargs['fromdate'] = fromdate
if args.todate:
todate = datetime.datetime.strptime(args.todate, '%Y-%m-%d')
dkwargs['todate'] = todate
data0 = bt.feeds.BacktraderCSVData(dataname=args.data0, **dkwargs)
cerebro.adddata(data0, name='Data0')
data1 = bt.feeds.BacktraderCSVData(dataname=args.data1, **dkwargs)
cerebro.adddata(data1, name='Data1')
data2 = bt.feeds.BacktraderCSVData(dataname=args.data2, **dkwargs)
cerebro.adddata(data2, name='Data2')
cerebro.addstrategy(St, printout=args.printout)
if not args.no_pyfolio:
cerebro.addanalyzer(bt.analyzers.PyFolio, _name='pyfolio')
results = cerebro.run()
if not args.no_pyfolio:
strat = results[0]
pyfoliozer = strat.analyzers.getbyname('pyfolio')
returns, positions, transactions, gross_lev = pyfoliozer.get_pf_items()
if args.printout:
print('-- RETURNS')
print(returns)
print('-- POSITIONS')
print(positions)
print('-- TRANSACTIONS')
print(transactions)
print('-- GROSS LEVERAGE')
print(gross_lev)
import pyfolio as pf
pf.create_full_tear_sheet(
returns,
positions=positions,
transactions=transactions,
gross_lev=gross_lev,
live_start_date='2005-05-01',
round_trips=True)
if args.plot:
cerebro.plot(style=args.plot_style)
def parse_args(args=None):
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description='Sample for pivot point and cross plotting')
parser.add_argument('--data0', required=False,
default='../../datas/yhoo-1996-2015.txt',
help='Data to be read in')
parser.add_argument('--data1', required=False,
default='../../datas/orcl-1995-2014.txt',
help='Data to be read in')
parser.add_argument('--data2', required=False,
default='../../datas/nvda-1999-2014.txt',
help='Data to be read in')
parser.add_argument('--fromdate', required=False,
default='2005-01-01',
help='Starting date in YYYY-MM-DD format')
parser.add_argument('--todate', required=False,
default='2006-12-31',
help='Ending date in YYYY-MM-DD format')
parser.add_argument('--printout', required=False, action='store_true',
help=('Print data lines'))
parser.add_argument('--cash', required=False, action='store',
type=float, default=50000,
help=('Cash to start with'))
parser.add_argument('--plot', required=False, action='store_true',
help=('Plot the result'))
parser.add_argument('--plot-style', required=False, action='store',
default='bar', choices=['bar', 'candle', 'line'],
help=('Plot style'))
parser.add_argument('--no-pyfolio', required=False, action='store_true',
help=('Do not do pyfolio things'))
import sys
aargs = args if args is not None else sys.argv[1:]
return parser.parse_args(aargs)
runstrat([])
However, my output is different to the one in the link (the values of the statistics, and produces some error messages along the way which I believe is preventing some of the statistics from being calculated):
Entire data start date: 2005-01-03
Entire data end date: 2006-12-29
Out-of-Sample Months: 20
Backtest Months: 3
Performance statistics All history Backtest Out of sample
annual_return -0.01 -0.07 0.00
annual_volatility 0.23 0.57 0.00
sharpe_ratio 0.05 0.12 0.22
calmar_ratio -0.05 -0.30 0.14
stability_of_timeseries -0.19 -0.41 -0.15
max_drawdown -0.24 -0.24 -0.00
omega_ratio 1.08 1.08 1.06
sortino_ratio 0.11 0.28 0.25
skew 15.26 6.14 -9.40
kurtosis 339.59 52.70 145.92
tail_ratio 0.77 0.06 1.05
common_sense_ratio 0.76 0.06 1.05
information_ratio -0.02 0.02 -0.09
alpha 0.01 0.07 -0.00
beta 0.00 0.02 0.00
/Users/maxpaton/anaconda3/lib/python3.6/site-packages/numpy/core/fromnumeric.py:52: FutureWarning: 'argmin' is deprecated, use 'idxmin' instead. The behavior of 'argmin'
will be corrected to return the positional minimum in the future.
Use 'series.values.argmin' to get the position of the minimum now.
return getattr(obj, method)(*args, **kwds)
Worst Drawdown Periods net drawdown in % peak date valley date recovery date duration
0 24.27 2005-01-10 2005-01-25 NaT NaN
1 0.80 2005-01-03 2005-01-04 2005-01-07 5
2 0.00 2005-01-03 2005-01-03 2005-01-03 1
3 0.00 2005-01-03 2005-01-03 2005-01-03 1
4 0.00 2005-01-03 2005-01-03 2005-01-03 1
[-0.029 -0.069]
/Users/maxpaton/anaconda3/lib/python3.6/site-packages/seaborn/categorical.py:454: FutureWarning: remove_na is deprecated and is a private function. Do not use.
box_data = remove_na(group_data)
Stress Events mean min max
Low Volatility Bull Market 0.00% -11.26% 29.32%
Top 10 long positions of all time max
Data2 35.47%
Empty DataFrame
Columns: [max]
Index: []
Top 10 positions of all time max
Data2 35.47%
All positions ever held max
Data2 35.47%
/Users/maxpaton/anaconda3/lib/python3.6/site-packages/matplotlib/axes/_axes.py:6462: UserWarning: The 'normed' kwarg is deprecated, and has been replaced by the 'density' kwarg.
warnings.warn("The 'normed' kwarg is deprecated, and has been "
/Users/maxpaton/anaconda3/lib/python3.6/site-packages/pyfolio/round_trips.py:127: FutureWarning: Interpreting tuple 'by' as a list of keys, rather than a single key. Use 'by=[...]' instead of 'by=(...)'. In the future, a tuple will always mean a single key.
'block_time'))
/Users/maxpaton/anaconda3/lib/python3.6/site-packages/pyfolio/round_trips.py:130: FutureWarning: Interpreting tuple 'by' as a list of keys, rather than a single key. Use 'by=[...]' instead of 'by=(...)'. In the future, a tuple will always mean a single key.
grouped_rest = t.groupby(('block_dir', 'block_time')).agg({
Traceback (most recent call last):
File "BT_pyfolio_eg.py", line 177, in <module>
runstrat([])
File "BT_pyfolio_eg.py", line 120, in runstrat
round_trips=True)
File "/Users/maxpaton/anaconda3/lib/python3.6/site-packages/pyfolio/tears.py", line 192, in create_full_tear_sheet
sector_mappings=sector_mappings)
File "/Users/maxpaton/anaconda3/lib/python3.6/site-packages/pyfolio/plotting.py", line 49, in call_w_context
return func(*args, **kwargs)
File "/Users/maxpaton/anaconda3/lib/python3.6/site-packages/pyfolio/tears.py", line 584, in create_round_trip_tear_sheet
portfolio_value=positions.sum(axis='columns') / (1 + returns)
File "/Users/maxpaton/anaconda3/lib/python3.6/site-packages/pyfolio/round_trips.py", line 261, in extract_round_trips
.join(pv, on='date', lsuffix='_')
File "/Users/maxpaton/anaconda3/lib/python3.6/site-packages/pandas/core/frame.py", line 6326, in join
rsuffix=rsuffix, sort=sort)
File "/Users/maxpaton/anaconda3/lib/python3.6/site-packages/pandas/core/frame.py", line 6341, in _join_compat
suffixes=(lsuffix, rsuffix), sort=sort)
File "/Users/maxpaton/anaconda3/lib/python3.6/site-packages/pandas/core/reshape/merge.py", line 60, in merge
validate=validate)
File "/Users/maxpaton/anaconda3/lib/python3.6/site-packages/pandas/core/reshape/merge.py", line 554, in __init__
self._maybe_coerce_merge_keys()
File "/Users/maxpaton/anaconda3/lib/python3.6/site-packages/pandas/core/reshape/merge.py", line 986, in _maybe_coerce_merge_keys
raise ValueError(msg)
ValueError: You are trying to merge on datetime64[ns, UTC] and datetime64[ns] columns. If you wish to proceed you should use pd.concat
Please could you tell me why this is, and suggest a solution?
@backtrader Got it now, thanks
I have developed my first custom backtrader indicator class (in a separate file) which calculates resistance values. The first thing I do in the next() method of this indicator class is to get the previous 8 values of self.data and store them in an array named 'ltp', like so:
class supres(bt.Indicator):
lines = ('support','resistance')
params = (('n', 8),)
def __init__(self):
#converting n to a nearest even number
if self.p.n%2 != 0:
self.p.n += 1
def next(self):
ltp = self.data.get(ago=0, size=self.p.n)
print(ltp)
When I come to call the indicator in the init of my strategy, I pass in self.data[0]. But the self.data[0] doesn't seem to be available in the the indicator class. When I print out 'ltp' from the indicator class the array is empty.
Here is my strategy code for the init and main that isolates the problem:
import backtrader as bt
import datetime
from backtrader import indicators as ind
import numpy as np
from supres_ind import supres
# Create strategy
class TestStrategy(bt.Strategy):
def __init__(self):
# Keep a reference to the "close" line in the data[0] dataseries
self.dataclose = self.datas[0].close
#calls the indicator
self.support = supres(self.datas[0])
and
if __name__ == '__main__':
datapath = '../../data/raw_bitcoin_data_copy.csv'
data = bt.feeds.BacktraderCSVData(
dataname=datapath,
fromdate=datetime.datetime(2017, 1, 1),
todate=datetime.datetime(2017, 7, 1),
adjclose=False)
cerebro = bt.Cerebro()
cerebro.adddata(data)
cerebro.addstrategy(TestStrategy)
cerebro.broker.setcash(100000.0)
cerebro.addsizer(bt.sizers.SizerFix, stake=10)
cerebro.broker.setcommission(0.001) #0.1%
print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue())
cerebro.run()
print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue())
Do I have to add the indicator class to the cerebro engine somehow? And does the indicator have to be called with bt.ind prefix?
Here is the form of the data:
timestamp open high low close volume adj close
2017-01-01 973.37 1000 964.37 992.95 4421.502883 992.95
2017-01-02 992.96 1034.39 990.52 1011.45 7723.635752 1011.45
2017-01-03 1011.52 1036.99 1006.71 1020.67 8614.250058 1020.67
2017-01-04 1020.69 1147 1018 1130.3 16222.23027 1130.3
2017-01-05 1131.1 1175 880 1007 20438.58193 1007
2017-01-06 1007 1033.85 875.01 895.71 14262.50427 895.71
Cheers
@backtrader It doesn't seem like you've missed anything, no. It had occurred to me that I could place the secondary order in notify_order(), but I just thought you couldn't as I had never seen done it anywhere else.
It looks like I should be able to get my algo working by placing the secondary order (on occurrence of order.Completed), in notify_order()!
Thanks for the clarification, and will let you know if I have any further problems in relation to this post.
@samy-david I understand that, and already have notify_order in my strategy. Currently, use it to log various details of orders depending on their order.status.
However, I place all orders in the next() method, and can't seem to get the order.status whilst in next(). It seems I need a way of using the order.status in order to trigger the second trade (i.e. when the previous limit order is executed)
Hi,
I currently place a market order, along with a limit order and stop order (of different sizes), all at the same time in next(). If that limit price is reached, I want to immediately place a new limit and stop order (of different sizes) based around this limit price.
Please could you tell me the best way of doing this?
@backtrader Yes I did mean notify_order, thanks. I agree, not sure why I added the code, as all I wanted to know was how order.executed.value is calculated.
Cheers
@backtrader Sorry, I have realised the mistake I was making. But, would it be possible just to tell me how order.execute.value is calculated in the notify trade() method?
Here is the code:
def notify_order(self, order):
if order.status in [order.Submitted, order.Accepted]:
return
if order.status in [order.Completed]:
if order.isbuy():
self.log('BUY EXECUTED, Price: %.2f, Cost: %.2f, Comm: %.2f' %
(order.executed.price,
order.executed.value,
order.executed.comm))
self.buyprice = order.executed.price
self.buycomm = order.executed.comm
else:
self.log('SELL EXECUTED, Price: %.2f, Cost: %.2f, Comm: %.2f' %
(order.executed.price,
order.executed.value,
order.executed.comm))
self.bar_executed = len(self)
elif order.status in [order.Cancelled, order.Margin, order.Rejected]:
self.log('Order Cancelled/Margin/Rejected')Taking the QuickStart section as an example, assuming the a stake size of 1, how is order.executed.price different from order.executed.value? In the Quickstart section it logs order.executed.value as 'cost'.
When I use it in work of my own in the same way, order.executed.value ('cost') is sometimes the same value as order.executed.price ('price'), and other times it differs slightly...
I have defined two SMA's in my init like so:
def log(self, txt, dt=None):
''' Logging function for this strategy'''
dt = dt or self.datas[0].datetime.date(0)
print('%s, %s' % (dt.isoformat(), txt))
def __init__(self):
self.dataclose = self.datas[0].close
self.sma1 = bt.ind.SMA(self.datas[0], period = 8)
self.sma2 = bt.ind.SMA(self.datas[0], period = 21)
self.trade_hist = []
as well as an empty list 'self.trade_hist' in order to accumulate all close prices at each 'next'.
I append the close prices to this list in the 'next' method, like so:
self.trade_hist.append(self.dataclose[0])
When I log 'self.trade_hist' in the 'next' method, the list only starts filling up with the close prices starting from the 21st data point (i.e. after the slowest SMA kicks in). This is the output:
2017-01-21, [925.06]
2017-01-22, [925.06, 931.22]
2017-01-23, [925.06, 931.22, 910.7]
2017-01-24, [925.06, 931.22, 910.7, 887.0]
2017-01-25, [925.06, 931.22, 910.7, 887.0, 896.0]
and so on... I want to append the close prices starting from the first data point (2017-01-01), before the SMA's kick in.
Cheers