[Blotter-commits] r1748 - pkg/quantstrat/sandbox/backtest_musings
noreply at r-forge.r-project.org
noreply at r-forge.r-project.org
Sun Apr 17 20:25:29 CEST 2016
Author: braverock
Date: 2016-04-17 20:25:28 +0200 (Sun, 17 Apr 2016)
New Revision: 1748
Modified:
pkg/quantstrat/sandbox/backtest_musings/stat_process.bib
pkg/quantstrat/sandbox/backtest_musings/strat_dev_process.Rmd
pkg/quantstrat/sandbox/backtest_musings/strat_dev_process.pdf
Log:
- update public version of strategy development process essay
Modified: pkg/quantstrat/sandbox/backtest_musings/stat_process.bib
===================================================================
--- pkg/quantstrat/sandbox/backtest_musings/stat_process.bib 2016-04-09 13:18:05 UTC (rev 1747)
+++ pkg/quantstrat/sandbox/backtest_musings/stat_process.bib 2016-04-17 18:25:28 UTC (rev 1748)
@@ -1,4 +1,3 @@
-% This file was created with JabRef 2.10.
% Encoding: UTF-8
@@ -12,6 +11,24 @@
Author_sort = {Aronson, David}
}
+ at Article{Bailey2014deSharpe,
+ Title = {The Deflated Sharpe Ratio: Correcting for Selection Bias, Backtest Overfitting and Non-Normality},
+ Author = {Bailey, David H and {L{\'o}pez de Prado}, Marcos},
+ Journal = {Journal of Portfolio Management, Forthcoming},
+ Year = {2014},
+
+ Url = {http://www.davidhbailey.com/dhbpapers/deflated-sharpe.pdf}
+}
+
+ at Article{Bailey2014drawdown,
+ Title = {Drawdown-Based Stop-Outs and the 'Triple Penance' Rule},
+ Author = {Bailey, David H and {L{\'o}pez de Prado}, Marcos},
+ Journal = {Journal of Risk},
+ Year = {forthcoming, 2014},
+
+ Url = {http://papers.ssrn.com/sol3/papers.cfm?abstract_id=2201302}
+}
+
@Article{Bailey2014pm,
Title = {Pseudomathematics and financial charlatanism: The effects of backtest {O}ver fitting on out-of-sample performance},
Author = {Bailey, David H and Borwein, Jonathan M and {L{\'o}pez de Prado}, Marcos and Zhu, Qiji Jim},
@@ -30,24 +47,6 @@
Url = {http://papers.ssrn.com/sol3/papers.cfm?abstract_id=2326253}
}
- at Article{Bailey2014drawdown,
- Title = {Drawdown-Based Stop-Outs and the 'Triple Penance' Rule},
- Author = {Bailey, David H and {L{\'o}pez de Prado}, Marcos},
- Journal = {Journal of Risk},
- Year = {forthcoming, 2014},
-
- Url = {http://papers.ssrn.com/sol3/papers.cfm?abstract_id=2201302}
-}
-
- at Article{Bailey2014deSharpe,
- Title = {The Deflated Sharpe Ratio: Correcting for Selection Bias, Backtest Overfitting and Non-Normality},
- Author = {Bailey, David H and {L{\'o}pez de Prado}, Marcos},
- Journal = {Journal of Portfolio Management, Forthcoming},
- Year = {2014},
-
- Url = {http://www.davidhbailey.com/dhbpapers/deflated-sharpe.pdf}
-}
-
@Article{baquero2005,
Title = {Survival, look-ahead bias, and persistence in hedge fund performance},
Author = {Baquero, Guillermo and Ter Horst, Jenke and Verbeek, Marno},
@@ -109,13 +108,21 @@
Pages = {2079--2107},
Volume = {11},
- __markedentry = {[brian:]},
Owner = {brian},
Publisher = {JMLR. org},
Timestamp = {2015.06.10},
Url = {http://www.jmlr.org/papers/volume11/cawley10a/cawley10a.pdf}
}
+ at Article{Chang2015,
+ author = {Chang, Andrew C and Li, Phillip},
+ title = {Is Economics Research Replicable? Sixty Published Papers from Thirteen Journals Say'Usually Not'},
+ year = {2015},
+ __markedentry = {[brian:6]},
+ publisher = {FEDS Working Paper},
+ url = {https://www.federalreserve.gov/econresdata/feds/2015/files/2015083pap.pdf}
+}
+
@Article{Diedesch2014,
Title = {2014 Forty Under Forty},
Author = {Diedesch, Josh},
@@ -126,6 +133,18 @@
Url = {http://www.ai-cio.com/Forty_Under_Forty_2014.aspx?page=9}
}
+ at Article{Economist2013,
+ Title = {Unreliable Research: Trouble at the lab},
+ Journal = {Economist},
+ Year = {2013},
+
+ Month = {Oct 19},
+
+ Owner = {brian},
+ Timestamp = {2015.01.14},
+ Url = {http://www.economist.com/news/briefing/21588057-scientists-think-science-self-correcting-alarming-degree-it-not-trouble}
+}
+
@Book{Feynman1965,
Title = {The Feynman Lectures on Physics},
Author = {Feynman, Richard P and Leighton, Robert B and Sands, Matthew and Hafner, EM},
@@ -139,7 +158,7 @@
Publisher = {John Wiley \& Sons, Inc.},
Year = {2013},
- Subtitle = {Tradable Strategies That Perform as They Backtest and Meet Your Risk-Reward Goals}
+ Ssubtitle = {Tradable Strategies That Perform as They Backtest and Meet Your Risk-Reward Goals}
}
@Misc{Fox2011,
@@ -153,11 +172,14 @@
Timestamp = {2015.01.13}
}
- at Manual{parma2014,
- Title = {parma: portfolio allocation and risk management applications.},
- Author = {Alexios Ghalanos and Bernhard Pfaff},
- Note = {R package version 1.5-1.},
- Year = {2014}
+ at Book{goldman1998practice,
+ Title = {The practice of risk management},
+ Author = {Litterman, Robert and Gumerlock, Robert },
+ Publisher = {Euromoney},
+ Year = {1998},
+
+ Authorsort = {Litterman, Robert and Gumerlock, Robert},
+ Ssubtitle = {Implementing processes for managing firmwide market risk}
}
@Article{Hansen2005,
@@ -167,19 +189,6 @@
Year = {2005}
}
- at Article{Harvey2014,
- Title = {Evaluating Trading Strategies},
- Author = {Harvey, Campbell R. and Liu, Yan},
- Journal = {Journal of Portfolio Management},
- Year = {2014},
- Number = {5},
- Pages = {108-118},
- Volume = {40},
-
- Comment = {preprint at http://ssrn.com/abstract=2474755},
- Url = {https://faculty.fuqua.duke.edu/~charvey/Research/Published_Papers/P116_Evaluating_trading_strategies.pdf}
-}
-
@Article{Harvey2013backtesting,
Title = {Backtesting},
Author = {Harvey, Campbell R. and Liu, Yan},
@@ -198,6 +207,19 @@
Url = {http://ssrn.com/abstract=2358214}
}
+ at Article{Harvey2014,
+ Title = {Evaluating Trading Strategies},
+ Author = {Harvey, Campbell R. and Liu, Yan},
+ Journal = {Journal of Portfolio Management},
+ Year = {2014},
+ Number = {5},
+ Pages = {108-118},
+ Volume = {40},
+
+ Comment = {preprint at http://ssrn.com/abstract=2474755},
+ Url = {https://faculty.fuqua.duke.edu/~charvey/Research/Published_Papers/P116_Evaluating_trading_strategies.pdf}
+}
+
@Book{Hastie2009,
Title = {The elements of statistical learning: Data mining, inference, and prediction. Second Edition},
Author = {Hastie, Trevor and Tibshirani, Robert and Friedman, Jerome},
@@ -244,12 +266,23 @@
Number = {8},
Pages = {e124},
Volume = {2},
+
Owner = {brian},
Publisher = {Public Library of Science},
Timestamp = {2015.01.29},
Url = {http://journals.plos.org/plosmedicine/article?id=10.1371/journal.pmed.0020124#s6}
}
+ at Book{Jones1999,
+ Title = {The trading game},
+ Author = {Jones, Ryan},
+ Publisher = {John Wiley \& Sons},
+ Year = {1999},
+ Owner = {brian},
+ Ssubtitle = {playing by the numbers to make millions},
+ Timestamp = {2015.08.13}
+}
+
@Article{Kaastra1996,
Title = {Designing a neural network for forecasting financial and economic time series},
Author = {Kaastra, Iebeling and Boyd, Milton},
@@ -264,6 +297,17 @@
Timestamp = {2015.05.19}
}
+ at Book{Kaufman2013,
+ Title = {Trading systems and methods},
+ Author = {Kaufman, Perry J},
+ Publisher = {John Wiley \& Sons},
+ Year = {2013},
+ Edition = {Fifth ed.},
+
+ Owner = {brian},
+ Timestamp = {2015.08.12}
+}
+
@Article{Keogh2003,
Title = {On the need for time series data mining benchmarks: a survey and empirical demonstration},
Author = {Keogh, Eamonn and Kasetty, Shruti},
@@ -273,13 +317,28 @@
Pages = {349--371},
Volume = {7},
- __markedentry = {[brian:6]},
Owner = {brian},
Publisher = {Springer},
Timestamp = {2015.06.10},
Url = {http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.13.2240&rep=rep1&type=pdf}
}
+ at Article{Kerr1998,
+ author = {Kerr, Norbert L},
+ title = {HARKing: Hypothesizing after the results are known},
+ journal = {Personality and Social Psychology Review},
+ year = {1998},
+ volume = {2},
+ number = {3},
+ pages = {196--217},
+ __markedentry = {[brian:]},
+ file = {:/home/brian/docs/Research/HARKing_ Hypothesizing_After_the_Results_are_Known_1998_Kerr.pdf:PDF},
+ owner = {brian},
+ publisher = {Sage Publications},
+ timestamp = {2016.02.25},
+ url = {http://psr.sagepub.com/content/2/3/196.abstract}
+}
+
@Book{Kestner2003,
Title = {Quantitative trading strategies: {H}arnessing the power of quantitative techniques to create a winning trading program},
Author = {Kestner, Lars},
@@ -313,19 +372,6 @@
Url = {http://appliedpredictivemodeling.com/}
}
- at InProceedings{Sweave2002,
- Title = {Sweave: Dynamic Generation of Statistical Reports Using Literate Data Analysis},
- Author = {Friedrich Leisch},
- Booktitle = {Compstat 2002 --- Proceedings in Computational Statistics},
- Year = {2002},
- Editor = {Wolfgang H{\"a}rdle and Bernd R{\"o}nz},
- Note = {ISBN 3-7908-1517-9},
- Pages = {575--580},
- Publisher = {Physica Verlag, Heidelberg},
-
- Url = {http://www.stat.uni-muenchen.de/~leisch/Sweave}
-}
-
@Article{Levy2006,
Title = {A systems approach to conduct an effective literature review in support of information systems research},
Author = {Levy, Yair and Ellis, Timothy J},
@@ -341,6 +387,29 @@
Url = {http://inform.nu/Articles/Vol9/V9p181-212Levy99.pdf}
}
+ at Article{Markowitz1994,
+ author = {Markowitz, Harry M and Xu, Gan Lin},
+ title = {Data mining corrections},
+ journal = {The Journal of Portfolio Management},
+ year = {1994},
+ volume = {21},
+ number = {1},
+ pages = {60--69},
+ owner = {brian},
+ publisher = {Institutional Investor Journals},
+ timestamp = {2015.08.19}
+}
+
+ at Misc{mistakes2011,
+ Title = {Common ~~ misteaks ~~ mistakes in using statistics: Spotting and Avoiding Them - Data Snooping},
+
+ Author = {Martha K. Smith},
+
+ Accessed = {2014-09-23},
+ Author_sort = {Smith, Martha},
+ Url = {https://www.ma.utexas.edu/users/mks/statmistakes/datasnooping.html}
+}
+
@Article{Moonesinghe2007,
Title = {Most published research findings are false, but a little replication goes a long way},
Author = {Moonesinghe, Ramal and Khoury, Muin J and Janssens, A Cecile JW},
@@ -356,16 +425,34 @@
Url = {http://journals.plos.org/plosmedicine/article?id=info:doi/10.1371/journal.pmed.0040028}
}
+ at Book{Narang2013,
+ Title = {Inside the Black Box: A Simple Guide to Quantitative and High Frequency Trading},
+ Author = {Narang, Rishi K},
+ Publisher = {John Wiley \& Sons},
+ Year = {2013},
+
+ Owner = {brian},
+ Timestamp = {2015.06.18}
+}
+
@Book{Pardo2008,
- Title = {The evaluation and optimization of trading strategies, second edition},
+ Title = {The evaluation and optimization of trading strategies},
Author = {Robert Pardo},
Publisher = {John Wiley \& Sons},
Year = {2008},
+ Edition = {Second ed.},
Month = {Feb},
Author_sort = {Pardo, Robert}
}
+ at Manual{parma2014,
+ Title = {parma: portfolio allocation and risk management applications.},
+ Author = {Alexios Ghalanos and Bernhard Pfaff},
+ Note = {R package version 1.5-1.},
+ Year = {2014}
+}
+
@Article{Peng2011,
Title = {Reproducible research in computational science},
Author = {Peng, Roger D},
@@ -381,6 +468,15 @@
Url = {http://www.ncbi.nlm.nih.gov/pmc/articles/PMC3383002/}
}
+ at Manual{perfa2014,
+ Title = {PerformanceAnalytics: Econometric tools for performance and risk analysis},
+ Author = {Brian G. Peterson and Peter Carl},
+ Year = {2015},
+
+ Subtitle = {R package version 1.4.3541},
+ Url = {http://CRAN.R-project.org/package=PerformanceAnalytics}
+}
+
@TechReport{Peterson2015,
Title = {Developing \& Backtesting Systematic Trading Strategies},
Author = {Peterson, Brian G},
@@ -390,15 +486,6 @@
Url = {http://goo.gl/na4u5d}
}
- at Manual{perfa2014,
- Title = {PerformanceAnalytics: Econometric tools for performance and risk analysis},
- Author = {Brian G. Peterson and Peter Carl},
- Year = {2015},
-
- Subtitle = {R package version 1.4.3541},
- Url = {http://CRAN.R-project.org/package=PerformanceAnalytics}
-}
-
@Manual{porta2014,
Title = {PortfolioAnalytics: Portfolio Analysis, including Numerical Methods for Optimization of Portfolios},
Author = {Brian G. Peterson and Peter Carl and Ross Bennett and Kris Boudt},
@@ -445,14 +532,15 @@
Pages = {155--170}
}
- at Misc{mistakes2011,
- Title = {Common ~~ misteaks ~~ mistakes in using statistics: Spotting and Avoiding Them - Data Snooping},
+ at Misc{Rmarkdown,
+ Title = {R Markdown — Dynamic Documents for R},
- Author = {Martha K. Smith},
+ Author = {Yihui Xie},
+ Year = {2014},
- Accessed = {2014-09-23},
- Author_sort = {Smith, Martha},
- Url = {https://www.ma.utexas.edu/users/mks/statmistakes/datasnooping.html}
+ Owner = {brian},
+ Timestamp = {2015.01.13},
+ Url = {http://rmarkdown.rstudio.com/}
}
@Article{Sullivan1999,
@@ -465,16 +553,27 @@
Volume = {54}
}
- at Misc{Tomasini2009,
+ at InProceedings{Sweave2002,
+ Title = {Sweave: Dynamic Generation of Statistical Reports Using Literate Data Analysis},
+ Author = {Friedrich Leisch},
+ Booktitle = {Compstat 2002 --- Proceedings in Computational Statistics},
+ Year = {2002},
+ Editor = {Wolfgang H{\"a}rdle and Bernd R{\"o}nz},
+ Note = {ISBN 3-7908-1517-9},
+ Pages = {575--580},
+ Publisher = {Physica Verlag, Heidelberg},
+
+ Url = {http://www.stat.uni-muenchen.de/~leisch/Sweave}
+}
+
+ at Book{Tomasini2009,
Title = {Trading Systems: A New Approach to System Development and Portfolio Optimisation },
-
Author = {Emilio Tomasini and Urban Jaekle},
+ Year = {2009},
Month = {Dec},
- Year = {2009},
Author_sort = {Tomasini, Emilio \& Jaekle, Urban},
- Title_sort = {Trading Systems A New Approach to System Development and Portfolio Optimisation},
- Volume = {1}
+ Title_sort = {Trading Systems A New Approach to System Development and Portfolio Optimisation}
}
@Article{Tukey1962,
@@ -488,12 +587,22 @@
Url = {http://projecteuclid.org/euclid.aoms/1177704711}
}
+ at Book{Vince1992,
+ Title = {The mathematics of money management},
+ Author = {Vince, Ralph},
+ Publisher = {John Wiley \& Sons},
+ Year = {1992},
+
+ Owner = {brian},
+ Ssubtitle = {Risk Analysis Techniques for traders},
+ Timestamp = {2015.08.12}
+}
+
@Book{Vince2009,
Title = {The leverage space trading model},
Author = {Vince, Ralph},
- Publisher = {John Wiley and Sons},
+ Publisher = {John Wiley \& Sons},
Year = {2009},
- Volume = {425},
Subtitle = {Reconciling portfolio management strategies and economic theory}
}
@@ -509,6 +618,17 @@
Url = {http://openeconomics.net/resources/data-policies-of-economic-journals/}
}
+ at Book{Weissman2005,
+ Title = {Mechanical trading systems},
+ Author = {Weissman, Richard L},
+ Publisher = {John Wiley \& Sons},
+ Year = {2005},
+
+ Owner = {brian},
+ Ssubtitle = {Pairing trader psychology with technical analysis},
+ Timestamp = {2015.08.12}
+}
+
@Misc{White2000,
Title = {System and method for testing prediction models and/or entities},
@@ -522,26 +642,3 @@
Url = {http://www.google.com/patents/US6088676}
}
- at Misc{Rmarkdown,
- Title = {R Markdown — Dynamic Documents for R},
-
- Author = {Yihui Xie},
- Year = {2014},
-
- Owner = {brian},
- Timestamp = {2015.01.13},
- Url = {http://rmarkdown.rstudio.com/}
-}
-
- at Article{Economist2013,
- Title = {Unreliable Research: Trouble at the lab},
- Journal = {Economist},
- Year = {2013},
-
- Month = {Oct 19},
-
- Owner = {brian},
- Timestamp = {2015.01.14},
- Url = {http://www.economist.com/news/briefing/21588057-scientists-think-science-self-correcting-alarming-degree-it-not-trouble}
-}
-
Modified: pkg/quantstrat/sandbox/backtest_musings/strat_dev_process.Rmd
===================================================================
--- pkg/quantstrat/sandbox/backtest_musings/strat_dev_process.Rmd 2016-04-09 13:18:05 UTC (rev 1747)
+++ pkg/quantstrat/sandbox/backtest_musings/strat_dev_process.Rmd 2016-04-17 18:25:28 UTC (rev 1748)
@@ -3,7 +3,7 @@
author: Brian G. Peterson
bibliography: stat_process.bib
output:
- rmarkdown::tufte_handout
+ tufte::tufte_handout
#tufterhandout::html_tufte_handout
#toc: true
#toc_depth: 2
@@ -12,8 +12,8 @@
keywords: quantitative trading, backtest, quantitative strategy, scientific method
subject: quantitative trading, backtest, quantitative strategy, scientific method
-footer: Copyright 2014 Brian G. Peterson CC-BY-NC-SA.
-copyright: Copyright 2014 Brian G. Peterson CC-BY-NC-SA.
+footer: Copyright 2014-2016 Brian G. Peterson CC-BY-NC-SA.
+copyright: Copyright 2014-2016 Brian G. Peterson CC-BY-NC-SA.
abstract: Analysts and portfolio managers face many challenges in developing new systematic trading systems. This paper provides a detailed, repeatable process to aid in evaluating new ideas, developing those ideas into testable hypotheses, measuring results in comparable ways, and avoiding and measuring the ever-present risks of over-fitting. ^[ *Back-testing. I hate it –- it's just optimizing over history. You never see a bad back-test. Ever. In any strategy.* - Josh Diedesch[- at Diedesch2014] ]
@@ -322,6 +322,14 @@
multiple times while the model is "refined", introducing progressively
more data snooping bias.
+Social scientists have coined the term HARKing[@Kerr1998] for "hypothesizing
+after the results are known". This needs to be separated from true scientific
+inference, which develops an **a priori** hypothesis from examining raw or
+unmassaged data. Defining a hypothesis should proceed from data to question to
+testable conjecture about a dependent variable. Rejecting a hypothesis is more
+valuable that inventing one after the fact. (and avoids losing money on an random
+or over-fit model)
+
___
# Defining the strategy
@@ -334,17 +342,18 @@
*filters*, *indicators*, *signals*, and *rules*.
\newthought{Filters} help to select the instruments to trade.
-They may be part of the formulated hypothesis, or they may be market
-characteristics that allow the rest of the strategy to trade better.
-In fundamental equity investing, some strategies consist only of filters.
-For example, the StarMine package that was bought by Thomson Reuters defines
-quantitative stock screens based on technicals or fundamentals.^[a modern, free
-alternative may be found at http://finviz.com/screener.ashx]
-Many analysts will expand or shrink their investible universe based on screens.
-Lo's Variance Ratio is another measure often used as a filter to turn the
-strategy on or off for particular instruments (but can also be used as an
-indicator, since it is time-varying).
+They may be part of the formulated hypothesis, or they may be market
+characteristics that allow the rest of the strategy to trade better. In
+fundamental equity investing, some strategies consist only of filters. For
+example, the StarMine package that was bought by Thomson Reuters defines
+quantitative stock screens based on technicals or fundamentals.^[a modern, free
+alternative may be found at http://finviz.com/screener.ashx] Many analysts will
+expand or shrink their investible universe based on screens. Lo's Variance Ratio
+is another measure often used as a filter to turn the strategy on or off for
+particular instruments (but can also be used as an indicator, since it is
+time-varying).
+
\newthought{Indicators} are quantitative values derived from market data.
Examples of indicators include moving averages, RSI, MACD, volatility bands,
@@ -1115,7 +1124,7 @@
Flat to flat "trade" analysis marks the beginning of the trade from the
first transaction to move the position off zero, and marks the end of
- the "trade" with the transaction that brings the P\&L back to zero,
+ the "trade" with the transaction that brings the position back to zero,
or "flat".
It will match brokerage statements of realized P\&L when the positions is
flat and average cost of open positions always, so it is easy to reconcile
@@ -1157,9 +1166,14 @@
reduction to an increase, in expanding order from the first reduction.
This is "time and quantity" priority, and is analytically more repeatable
and manageable than FIFO. If you have a reason for utilizing FIFO-like
- analysis, consider using "increased to reduced" instead. Be aware that the
- "trade" statistics will not be reconcilable to any brokerage statement,
- this is purely an analytical methodology.
+ analysis, consider using "increased to reduced" instead. This methodology
+ is sometimes called "average cost FIFO", and there is an average cost LIFO
+ variant as well. In contrast to traditional FIFO, realized P\&L on a
+ per-period basis will match the brokerage statements using this methodology,
+ as realized P\&L is taken from the same average cost basis. Be aware that
+ the round-turn "trade" statistics may not be fully reconcilable to a
+ brokerage statement, as the separation into round-turn "trades" is purely an
+ analytical methodology.
\newthought{Aggregate trade statistics are calculated on the entire backtest.}
@@ -1734,7 +1748,7 @@
___
-©2014-2015 Brian G. Peterson
+©2014-2016 Brian G. Peterson
\includegraphics[width=1.75cm]{cc-by-nc-sa}
Modified: pkg/quantstrat/sandbox/backtest_musings/strat_dev_process.pdf
===================================================================
(Binary files differ)
More information about the Blotter-commits
mailing list