[Blotter-commits] r1690 - pkg/quantstrat/sandbox/backtest_musings
noreply at r-forge.r-project.org
noreply at r-forge.r-project.org
Wed Jun 10 23:43:33 CEST 2015
Author: braverock
Date: 2015-06-10 23:43:32 +0200 (Wed, 10 Jun 2015)
New Revision: 1690
Modified:
pkg/quantstrat/sandbox/backtest_musings/stat_process.bib
pkg/quantstrat/sandbox/backtest_musings/strat_dev_process.Rmd
pkg/quantstrat/sandbox/backtest_musings/strat_dev_process.pdf
Log:
- updates from UW CFRM 561 development
Modified: pkg/quantstrat/sandbox/backtest_musings/stat_process.bib
===================================================================
--- pkg/quantstrat/sandbox/backtest_musings/stat_process.bib 2015-06-10 09:37:03 UTC (rev 1689)
+++ pkg/quantstrat/sandbox/backtest_musings/stat_process.bib 2015-06-10 21:43:32 UTC (rev 1690)
@@ -76,19 +76,6 @@
Url = {http://www.nature.com/news/2010/101013/full/467753a.html}
}
- at Article{baxter1999,
- Title = {Measuring business cycles: approximate band-pass filters for economic time series},
- Author = {Baxter, Marianne and King, Robert G},
- Journal = {Review of economics and statistics},
- Year = {1999},
- Number = {4},
- Pages = {575--593},
- Volume = {81},
-
- Publisher = {MIT Press},
- Url = {http://pages.stern.nyu.edu/~dbackus/GE_asset_pricing/ms/Filters/BaxterKing%20bandpass%20NBER%205022.pdf}
-}
-
@Book{Box1987,
Title = {Empirical model-building and response surfaces.},
Author = {Box, George E.P. and Draper, Norman R.},
@@ -114,6 +101,21 @@
Url = {http://skepdic.com/adhoc.html}
}
+ at Article{Cawley2010,
+ Title = {On over-fitting in model selection and subsequent selection bias in performance evaluation},
+ Author = {Cawley, Gavin C and Talbot, Nicola LC},
+ Journal = {The Journal of Machine Learning Research},
+ Year = {2010},
+ Pages = {2079--2107},
+ Volume = {11},
+
+ __markedentry = {[brian:]},
+ Owner = {brian},
+ Publisher = {JMLR. org},
+ Timestamp = {2015.06.10},
+ Url = {http://www.jmlr.org/papers/volume11/cawley10a/cawley10a.pdf}
+}
+
@Article{Diedesch2014,
Title = {2014 Forty Under Forty},
Author = {Diedesch, Josh},
@@ -124,18 +126,6 @@
Url = {http://www.ai-cio.com/Forty_Under_Forty_2014.aspx?page=9}
}
- at Article{Dudler2014,
- Title = {Risk Adjusted Time Series Momentum},
- Author = {Dudler, Martin and Gmuer, Bruno and Malamud, Semyon},
- Journal = {Available at SSRN 2457647},
- Year = {2014},
-
- Abstract = {We introduce a new class of momentum strategies that are based on the long-term averages of risk-adjusted returns and test these strategies on a universe of 64 liquid futures contracts. We show that this risk adjusted momentum strategy outperforms the time series momentum strategy of Ooi, Moskowitz and Pedersen (2012) for almost all combinations of holding- and look-back periods. We construct measures of momentum-specific volatility (risk), (both within and across asset classes) and show that these volatility measures can be used both for risk management and it momentum timing. We find that momentum risk management significantly increases Sharpe ratios, but at the same time leads to more pronounced negative skewness and tail risk; by contrast, combining risk management with momentum timing practically eliminates the negative skewness of momentum returns and significantly reduces tail risk. In addition, momentum risk management leads to a much lower exposure to market, value, and momentum factors. As a result, risk-managed momentum returns offer much higher diversification benefits than the standard momentum returns.},
- Owner = {brian},
- Timestamp = {2015.01.15},
- Url = {http://papers.ssrn.com/sol3/papers.cfm?abstract_id=2457647}
-}
-
@Book{Feynman1965,
Title = {The Feynman Lectures on Physics},
Author = {Feynman, Richard P and Leighton, Robert B and Sands, Matthew and Hafner, EM},
@@ -254,27 +244,12 @@
Number = {8},
Pages = {e124},
Volume = {2},
-
- __markedentry = {[brian:6]},
Owner = {brian},
Publisher = {Public Library of Science},
Timestamp = {2015.01.29},
Url = {http://journals.plos.org/plosmedicine/article?id=10.1371/journal.pmed.0020124#s6}
}
- at Article{Jegadeesh2002,
- Title = {Cross-sectional and time-series determinants of momentum returns},
- Author = {Jegadeesh, Narasimhan and Titman, Sheridan},
- Journal = {Review of Financial Studies},
- Year = {2002},
- Number = {1},
- Pages = {143--157},
- Volume = {15},
-
- Publisher = {Soc Financial Studies},
- Url = {http://www.researchgate.net/profile/Narasimhan_Jegadeesh/publication/5216887_Cross-Sectional_and_Time-Series_Determinants_of_Momentum_Returns/links/0a85e5383ba5d2941e000000.pdf}
-}
-
@Article{Kaastra1996,
Title = {Designing a neural network for forecasting financial and economic time series},
Author = {Kaastra, Iebeling and Boyd, Milton},
@@ -289,6 +264,22 @@
Timestamp = {2015.05.19}
}
+ at Article{Keogh2003,
+ Title = {On the need for time series data mining benchmarks: a survey and empirical demonstration},
+ Author = {Keogh, Eamonn and Kasetty, Shruti},
+ Journal = {Data Mining and knowledge discovery},
+ Year = {2003},
+ Number = {4},
+ Pages = {349--371},
+ Volume = {7},
+
+ __markedentry = {[brian:6]},
+ Owner = {brian},
+ Publisher = {Springer},
+ Timestamp = {2015.06.10},
+ Url = {http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.13.2240&rep=rep1&type=pdf}
+}
+
@Book{Kestner2003,
Title = {Quantitative trading strategies: {H}arnessing the power of quantitative techniques to create a winning trading program},
Author = {Kestner, Lars},
@@ -359,7 +350,6 @@
Pages = {e28},
Volume = {4},
- __markedentry = {[brian:]},
Owner = {brian},
Publisher = {Public Library of Science},
Timestamp = {2015.01.29},
Modified: pkg/quantstrat/sandbox/backtest_musings/strat_dev_process.Rmd
===================================================================
--- pkg/quantstrat/sandbox/backtest_musings/strat_dev_process.Rmd 2015-06-10 09:37:03 UTC (rev 1689)
+++ pkg/quantstrat/sandbox/backtest_musings/strat_dev_process.Rmd 2015-06-10 21:43:32 UTC (rev 1690)
@@ -897,7 +897,7 @@
When a parameter or set of parameters is robust, it will have a few key
properties:
-- small parameter changes lead to small changes in P&L and objective
+- small parameter changes lead to small changes in P\&L and objective
expectations
- out of sample deterioration is not large, on average (see walk forward
optimization)
@@ -1072,10 +1072,10 @@
Entire books have been written extolling the virtues or lamenting the problems
of one performance measure over another. We have chosen to take a rather
-inclusive approach both to trade and P&L based measures and to return based
+inclusive approach both to trade and P\&L based measures and to return based
measures (covered later). Generally, we run as many metrics as we can, and
look for consistently good metrics across all common return and cash based
-measures. Trade and P&L based measures have an advantage of being precise
+measures. Trade and P\&L based measures have an advantage of being precise
and reconcilable to clearing statements, but disadvantages of not
being easily comparable between products, after compounding, etc.
@@ -1093,7 +1093,7 @@
FIFO is "first in, first out", and pairs entry and exit transactions by
time priority. We generally do not calculate statistics on FIFO because
- it is impossible to match P&L to clearing statements; very few
+ it is impossible to match P\&L to clearing statements; very few
institutional investors will track to FIFO. FIFO comes from accounting
for physical inventory, where old (first) inventory is accounted for in the
first sales of that inventory. It can be very difficult to calculate a cost
@@ -1114,9 +1114,9 @@
Flat to flat "trade" analysis marks the beginning of the trade from the
first transaction to move the position off zero, and marks the end of
- the "trade" with the transaction that brings the P&L back to zero,
+ the "trade" with the transaction that brings the P\&L back to zero,
or "flat".
- It will match brokerage statements of realized P&L when the positions is
+ It will match brokerage statements of realized P\&L when the positions is
flat and average cost of open positions always, so it is easy to reconcile
production trades using this methodology. One advantage of the flat to flat
methodology is that there are no overlapping "trades" so doing things like
@@ -1180,7 +1180,7 @@
- annualized Sharpe-like ratio
- max drawdown
- start-trade drawdown [@Fitschen2013, p. 185]
- - win/loss ratios of winning over losing trade P&L (total/mean/median)
+ - win/loss ratios of winning over losing trade P\&L (total/mean/median)
Some authors advocate testing the strategy against "perfect profit", the
@@ -1240,7 +1240,7 @@
Maximum Adverse Excursion (MAE) or Maximum Favorable Excursion (MFE) show how
far down (or up) every trade went during the course of its life-cycle. You
can capture information on how many trades close close to their highs or lows,
-as well as evaluating points at which the P&L for the trade statistically
+as well as evaluating points at which the P\&L for the trade statistically
just isn't going to get any better, or isn't going to recover. While
not useful for all strategies, many strategies will have clear patterns that
may be incorporated into risk or profit rules.
@@ -1279,7 +1279,7 @@
- modeled theoretical price (these should match in most cases)
- order timing and prices
-- P&L
+- P\&L
- evolution of the position, including partial fill analysis
Backtests tend to model complete fills, and use deliberately conservative
@@ -1346,7 +1346,7 @@
Returns based analysis is most valuable at daily or lower periodicities
(often requiring aggregating intraday transaction performance to daily returns)
, is easily comparable across assets or strategies, and has many analytical
-techniques not available for trades and P&L. Returns are not easily
+techniques not available for trades and P\&L. Returns are not easily
reconcilable back to the trades both because of the transformation done to
get to returns, changes in scale over time, and possible aggregation to lower
or regular frequencies different from the trading frequency.
@@ -1367,10 +1367,10 @@
\newthought{Should we use cash or percent returns?} If the strategy utilizes
the same order sizing in contract or cash terms throughout the tested period,
-then cash P&L should work fine. If instead the "money management" of the
+then cash P\&L should work fine. If instead the "money management" of the
strategy reinvests gains or changes order sizes based on account equity, then
you will need to resample percent returns, or your results will not be
-unbiased. The cash P&L would exhibit a (presumably upward) bias from the
+unbiased. The cash P\&L would exhibit a (presumably upward) bias from the
portfolio growth in the backtest portfolio over the course of the test if the
strategy is reinvesting. Conversely, if the strategy does not change order
sizes during the test (e.g. fixed 1-lot sizing), then using percent returns
@@ -1379,14 +1379,26 @@
a fixed/constant denominator to get to simple "returns", which would be the
equivalent of saying that any cash generated by the strategy was withdrawn.
+Many analytical techniques presume percent return-based analysis rather than
+analysis on cash P\&L. Some examples include:
- tail risk measures
- volatility analysis
- factor analysis
- factor model Monte Carlo
- style analysis
-- comparing strategies in return space
- applicability to asset allocation (see below)
+
+\newthought{Comparing strategies in return space} can also be a good reason to
+use percent returns rather than cash. When comparing strategies, worknig in
+return-space may allow for disparate strategies to be placed on a similar
+footing. Things like risk measures often make more sense when described
+against their percent impact on capital, for example.
+
+While it may be tempting to do all of the analysis of a trading strategy in
+only cash P&L, or only in return space, it is valuable to analyze almost every
+strategy in both ways, as the two approaches provide different insight.
+
___
# Rebalancing and asset allocation
@@ -1670,7 +1682,7 @@
evaluation with a rolling origin" is essentially Walk Forward Analysis. One
important takeaway from Prof. Hyndman's treatment of the subject is that it is
important to define the expected result and tests to measure forecast accuracy
-before performing the (back)test. Then, all the tools of forecast evaluation
+*before* performing the (back)test. Then, all the tools of forecast evaluation
may be applied to evaluate how well your forecast is doing out of sample, and
whether you are likely to have overfit your model.
@@ -1694,6 +1706,8 @@
## data mining bias
- data mining bias and cross validation from @Aronson2006
+- @Cawley2010
+- @Keogh2003
\newpage
Modified: pkg/quantstrat/sandbox/backtest_musings/strat_dev_process.pdf
===================================================================
(Binary files differ)
More information about the Blotter-commits
mailing list