[Blotter-commits] r610 - pkg/FinancialInstrument/sandbox
noreply at r-forge.r-project.org
noreply at r-forge.r-project.org
Thu Jun 2 04:11:13 CEST 2011
Author: peter_carl
Date: 2011-06-02 04:11:08 +0200 (Thu, 02 Jun 2011)
New Revision: 610
Added:
pkg/FinancialInstrument/sandbox/calc.GS10TR.R
pkg/FinancialInstrument/sandbox/download.DJUBSindex.R
pkg/FinancialInstrument/sandbox/download.MorningstarCLSIndex.R
pkg/FinancialInstrument/sandbox/download.NAREIT.R
pkg/FinancialInstrument/sandbox/parse.MSCI.R
pkg/FinancialInstrument/sandbox/parse.SP500TR.R
Log:
- drafts of scripts for downloading and parsing data for FI
- develop into demos, examples
Added: pkg/FinancialInstrument/sandbox/calc.GS10TR.R
===================================================================
--- pkg/FinancialInstrument/sandbox/calc.GS10TR.R (rev 0)
+++ pkg/FinancialInstrument/sandbox/calc.GS10TR.R 2011-06-02 02:11:08 UTC (rev 610)
@@ -0,0 +1,66 @@
+# Use yields of the Constant Maturity 10 year bond series from FRED
+# to calculate total returns.
+
+# Peter Carl
+
+# Originally described by Kenton Russell at his blog TimelyPortfolio,
+# posted April 15, 2011
+# http://timelyportfolio.blogspot.com/2011/04/historical-sources-of-bond-returns_17.html
+#
+
+require(quantmod)
+require(PerformanceAnalytics)
+require(RQuantLib)
+
+# Set the working directory.
+filesroot = "~/Data/FRED"
+
+# Create and set the working directory if it doesn't exist
+if (!file.exists(filesroot))
+ dir.create(filesroot, mode="0777")
+
+if (!file.exists(paste(filesroot, "/GS10TR.IDX", sep="")))
+ dir.create(paste(filesroot, "/GS10TR.IDX", sep=""), mode="0777")
+
+getSymbols("GS10", src="FRED") #load US Treasury 10y yields from FRED
+
+# Dates should be end of month, not beginning of the month as reported
+index(GS10) = as.Date(as.yearmon(index(GS10)), frac=1)
+
+# @TODO: Do this calculation with a longer list of symbols
+
+x.pr <- GS10 #set this up to hold price returns
+x.pr[1,1] <- 0
+colnames(x.pr) <- "Price Return"
+for (i in 1:(NROW(GS10)-1)) {
+ x.pr[i+1,1] <- FixedRateBondPriceByYield(yield=GS10[i+1,1]/100, issueDate=Sys.Date(), maturityDate=advance("UnitedStates/GovernmentBond", Sys.Date(), 10, 3), rates=GS10[i,1]/100,period=2)[1]/100-1
+}
+#total return will be the price return + yield/12 for one month
+x.tr <- x.pr + lag(GS10,k=1)/12/100
+colnames(x.tr)<-"Total Return"
+
+# Add an index column labeled "Close"
+x.idx = 100*cumprod(1+na.omit(x.tr))
+x.xts = cbind(x.idx, x.tr)
+x.xts[1,1]=100 # base the index at 100
+colnames(x.xts) = c("Close", "Return")
+
+# Save it into an rda file on the filesystem
+save(x.xts, file=paste(filesroot,"GS10TR.IDX/GS10TR.IDX.rda", sep="/"))
+
+# Create currencies first:
+require(FinancialInstrument)
+currency("USD")
+
+# Describe the metadata for the index
+instrument("GS10TR.IDX", currency="USD", multiplier=1, tick_size=.01, start_date="1953-04-01", description="US 10Y Constant Maturity Total Returns", data="CR", source="fred", assign_i=TRUE)
+
+# Now, whenever you log in you need to register the instruments. This
+# might be a line you put into .Rprofile so that it happens automatically:
+# require(quantmod) # this requires a development build after revision 560 or so.
+setSymbolLookup.FI(base_dir=filesroot, split_method='common')
+
+# Now you should be able to:
+getSymbols("GS10TR.IDX")
+chartSeries(Cl(GS10TR.IDX), theme="white")
+head(GS10TR.IDX)
Property changes on: pkg/FinancialInstrument/sandbox/calc.GS10TR.R
___________________________________________________________________
Added: svn:keywords
+ Revision Id Date Author
Added: pkg/FinancialInstrument/sandbox/download.DJUBSindex.R
===================================================================
--- pkg/FinancialInstrument/sandbox/download.DJUBSindex.R (rev 0)
+++ pkg/FinancialInstrument/sandbox/download.DJUBSindex.R 2011-06-02 02:11:08 UTC (rev 610)
@@ -0,0 +1,79 @@
+# Script for parsing DJUBS index daily price data series from the
+# DJ website.
+
+# Peter Carl
+
+# DETAILS
+# Parse index close prices from the spreadsheet containing the full series:
+# http://www.djindexes.com/mdsidx/downloads/xlspages/ubsci/DJUBS_full_hist.xls
+
+
+# Several series, all index values
+# Remove the footer at the bottom
+# Load needed packages:
+require(zoo)
+require(gdata)
+
+# Set the working directory, where there's a .incoming folder that contains
+# the downloaded spreadsheet.
+filesroot = "~/Data/DJUBS"
+
+# Create and set the working directory if it doesn't exist
+if (!file.exists(filesroot))
+ dir.create(filesroot, mode="0777")
+
+# Create and set the .incoming directory if it doesn't exist
+if (!file.exists(paste(filesroot, "/.incoming", sep="")))
+ dir.create(paste(filesroot, "/.incoming", sep=""), mode="0777")
+setwd(paste(filesroot, "/.incoming", sep=""))
+
+# Download the xls workbook directly from the web site:
+system("wget http://www.djindexes.com/mdsidx/downloads/xlspages/ubsci/DJUBS_full_hist.xls")
+
+if(!file.exists("DJUBS_full_hist.xls"))
+ stop(paste("No spreadsheet exists. Download the spreadsheet to be processed from http://www.msci.com into ", filesroot, "/.incoming", sep=""))
+
+x = read.xls("DJUBS_full_hist.xls", sheet="Total Return", pattern="Symbol")
+
+# Get the descriptions to add as attributes
+x.attr = read.xls("DJUBS_full_hist.xls", sheet="Total Return", pattern="Date", nrows=1, header=FALSE)
+
+# Get rid of the last line, which contains the disclaimer
+x=x[-dim(x)[1],]
+# Remove blank columns between sections
+x=x[,-which(apply(x,2,function(x)all(is.na(x))))]
+
+# Get attributes and labels
+categoryNames = x.attr[,!is.na(x.attr)]
+symbolNames = paste(make.names(colnames(x[,])), ".IDX", sep="")
+ISOdates = as.Date(x[,1], "%m/%d/%Y")
+
+for(i in 2:length(symbolNames)) {
+ # check to make sure directories exist for each symbol
+ dir.create(paste(filesroot, symbolNames[i], sep="/"), showWarnings = FALSE,
+ recursive = FALSE, mode = "0777")
+}
+
+# Parse the columns into individual price objects
+for( i in 2:dim(x)[2]){
+ x.xts = as.xts(as.numeric(x[,i]), order.by=ISOdates)
+ R.xts = Return.calculate(x.xts)
+ x.xts = cbind(x.xts, R.xts)
+ colnames(x.xts)=c("Close", "Returns")
+ xtsAttributes(x.xts) <- list(Description = paste(categoryNames[,i], "Index"))
+# assign(symbolNames[i], x.xts)
+ save(x.xts, file=paste(filesroot, symbolNames[i], paste(symbolNames[i], ".rda", sep=""), sep="/"))
+ # Describe the metadata for each index
+ instrument(symbolNames[i], currency="USD", multiplier=1, tick_size=.01, start_date=head(index(x.xts),1), description=paste(categoryNames[,i], "Index"), data="CR", source="DJUBS", assign_i=TRUE)
+}
+
+# Now, whenever you log in you need to register the instruments. This
+# might be a line you put into .Rprofile so that it happens automatically:
+# require(quantmod) # this requires a development build after revision 560 or so.
+setSymbolLookup.FI(base_dir=filesroot, split_method='common')
+
+# Now you should be able to:
+getSymbols("DJUBSTR.IDX")
+# chartSeries(Cl(DJUBSTR.IDX), theme="white")
+charts.PerformanceSummary(DJUBSTR.IDX["2010::","Returns"], ylog=TRUE, wealth.index=TRUE, main = "DJUBS Total Returns Index Returns")
+tail(DJUBSTR.IDX)
\ No newline at end of file
Property changes on: pkg/FinancialInstrument/sandbox/download.DJUBSindex.R
___________________________________________________________________
Added: svn:keywords
+ Revision Id Date Author
Added: pkg/FinancialInstrument/sandbox/download.MorningstarCLSIndex.R
===================================================================
--- pkg/FinancialInstrument/sandbox/download.MorningstarCLSIndex.R (rev 0)
+++ pkg/FinancialInstrument/sandbox/download.MorningstarCLSIndex.R 2011-06-02 02:11:08 UTC (rev 610)
@@ -0,0 +1,58 @@
+# Script for downloading and parsing Morningstar CLS Index
+# historical data series from Barclays website
+# http://www.barclayhedge.com/cgi-bin/barclay_stats/bcndx.cgi?dump=excel&daily=daily&prog_cod=morningSta
+
+# Peter Carl
+
+# DETAILS
+# Download and parse the Morningstar Long/Short Commodity Index
+
+# Load needed packages:
+require(zoo)
+require(gdata)
+require(quantmod)
+
+# Download the first sheet in the xls workbook directly from the web site:
+x = read.xls("http://www.barclayhedge.com/cgi-bin/barclay_stats/bcndx.cgi?dump=excel&daily=daily&prog_cod=morningSta", pattern="Date")
+# x = read.xls("/home/peter/Data/Commodity Returns.xls", sheet="Daily", pattern="Date")
+
+# That should result in something like this:
+# > head(x)
+# Date ROR VAMI
+# 1 Dec-26-1979 1.1753% 1011.75
+# 2 Dec-27-1979 0.5799% 1017.62
+# 3 Dec-28-1979 0.5103% 1022.81
+# 4 Dec-31-1979 0.7056% 1030.03
+# 5 Jan-01-1980 0.0000% 1030.03
+# 6 Jan-02-1980 0.9884% 1040.21
+
+# The first column is the date, in '%b-%d-%Y' format:
+ISOdates = as.Date(x[,1], "%b-%d-%Y")
+# ISOdates = as.Date(x[,"Date"], "%F") # new extract
+# The second column is the return, already calculated:
+ROR = as.numeric(gsub("[%]", "", x[,2]))/100
+
+# Create a return series
+x.R = xts(ROR, order.by=ISOdates)
+colnames(x.R)= "Returns"
+# Create a price series or index
+x.IDX = xts(x[,3], order.by=ISOdates)
+# x.IDX = xts(x[,"Index.Level"], order.by=ISOdates) # new extract
+colnames(x.IDX) = "Close"
+# new extract
+x.R = Return.calculate(x.IDX)
+colnames(x.R)= "Returns"
+# Merge the series into a single object
+MstarCLS.IDX = cbind(x.IDX, x.R)
+
+# Calculate monthly returns from daily index values
+x.M.IDX = to.monthly(Cl(MstarCLS.IDX))
+colnames(x.M.IDX)= c("Open", "High", "Low", "Close")
+x.M.R = Return.calculate(Cl(x.M.IDX))
+colnames(x.M.R)="Returns"
+MstarCLS.M.IDX = cbind(x.M.IDX, x.M.R)
+index(MstarCLS.M.IDX) <- as.Date(index(MstarCLS.M.IDX), frac=1)
+
+# Draw some pictures
+charts.PerformanceSummary(MstarCLS.IDX["2010::","Returns"], ylog=TRUE, wealth.index=TRUE, main = "Morningstar CLS Index Returns")
+MstarCLS.table=table.CalendarReturns(MstarCLS.M.IDX[,"Returns"])
Property changes on: pkg/FinancialInstrument/sandbox/download.MorningstarCLSIndex.R
___________________________________________________________________
Added: svn:keywords
+ Revision Id Date Author
Added: pkg/FinancialInstrument/sandbox/download.NAREIT.R
===================================================================
--- pkg/FinancialInstrument/sandbox/download.NAREIT.R (rev 0)
+++ pkg/FinancialInstrument/sandbox/download.NAREIT.R 2011-06-02 02:11:08 UTC (rev 610)
@@ -0,0 +1,54 @@
+# Script for downloading and parsing a monthly total return series of the FTSE NAREIT U.S. Real Estate Index
+# - Download the spreadsheet at:
+# http://returns.reit.com/returns/MonthlyHistoricalReturns.xls
+#
+
+# Peter Carl
+
+# Load needed packages:
+require(zoo)
+require(gdata)
+
+# Set the working directory, where there's a .incoming folder that contains
+# the downloaded spreadsheet.
+filesroot = "~/Data/NAREIT"
+
+# Create and set the working directory if it doesn't exist
+if (!file.exists(filesroot))
+ dir.create(filesroot, mode="0777")
+
+if (!file.exists(paste(filesroot, "/NAREIT.IDX", sep="")))
+ dir.create(paste(filesroot, "/NAREIT.IDX", sep=""), mode="0777")
+
+# Download the first sheet in the xls workbook directly from the web site:
+x = read.xls("http://returns.reit.com/returns/MonthlyHistoricalReturns.xls", pattern="Date", sheet="Data")
+
+# We'll focus on the first three columns, the date, the total return, and the total return index. The next columns divide it into price, income and dividend yield components, and after that are the individual sectors.
+# @TODO: create symbols for the rest of the columns in the spreadsheet
+x.dates = as.Date(as.yearmon(x[,1], format="%b-%y"), frac=1)
+x.returns = xts(x[,2]/100, order.by = x.dates)
+x.price = as.numeric((sub(",","", x[,3], fixed=TRUE))) # get rid of commas
+x.price = xts(x.price, order.by = x.dates)
+
+x.xts = cbind(x.price, x.returns)
+colnames(x.xts) = c("Close", "Returns")
+
+# Save it into an rda file on the filesystem
+save(x.xts, file=paste(filesroot,"NAREIT.IDX/NAREIT.IDX.rda", sep="/"))
+
+# Create currencies first:
+require(FinancialInstrument)
+currency("USD")
+
+# Describe the metadata for the index
+instrument("NAREIT.IDX", currency="USD", multiplier=1, tick_size=.01, start_date="1971-12-31", description="FTSE NAREIT U.S. Real Estate Index", data="CR", source="reit.com", assign_i=TRUE)
+
+# Now, whenever you log in you need to register the instruments. This
+# might be a line you put into .Rprofile so that it happens automatically:
+# require(quantmod) # this requires a development build after revision 560 or so.
+setSymbolLookup.FI(base_dir=filesroot, split_method='common')
+
+# Now you should be able to:
+getSymbols("NAREIT.IDX")
+chartSeries(Cl(NAREIT.IDX), theme="white")
+head(NAREIT.IDX)
Property changes on: pkg/FinancialInstrument/sandbox/download.NAREIT.R
___________________________________________________________________
Added: svn:keywords
+ Revision Id Date Author
Added: pkg/FinancialInstrument/sandbox/parse.MSCI.R
===================================================================
--- pkg/FinancialInstrument/sandbox/parse.MSCI.R (rev 0)
+++ pkg/FinancialInstrument/sandbox/parse.MSCI.R 2011-06-02 02:11:08 UTC (rev 610)
@@ -0,0 +1,63 @@
+# Script for downloading and parsing a monthly total return series from
+# http://www.mscibarra.com/
+#
+
+# Peter Carl
+
+# Currently, this script assumes that a single index is in the spreadsheet.
+# For cases where there are multiple indexes, this script will need to be
+# extended.
+
+# Load needed packages:
+require(zoo)
+require(gdata)
+
+# Set the working directory, where there's a .incoming folder that contains
+# the downloaded spreadsheet.
+filesroot = "~/Data/MSCI"
+
+# Create and set the working directory if it doesn't exist
+if (!file.exists(filesroot))
+ dir.create(filesroot, mode="0777")
+
+# Create and set the .incoming and symbol directories if they don't exist
+if (!file.exists(paste(filesroot, "/.incoming", sep="")))
+ dir.create(paste(filesroot, "/.incoming", sep=""), mode="0777")
+setwd(paste(filesroot, "/.incoming", sep=""))
+
+if (!file.exists(paste(filesroot, "/MSCI.World.IDX", sep="")))
+ dir.create(paste(filesroot, "/MSCI.World.IDX", sep=""), mode="0777")
+
+if(!file.exists("historyIndex.xls"))
+ stop(paste("No spreadsheet exists. Download the spreadsheet to be processed from http://www.msci.com into ", filesroot, "/.incoming", sep=""))
+
+# Read the first sheet in the xls workbook directly from the working directory:
+x = read.xls("historyIndex.xls", pattern="Date")
+x = x[-((dim(x)[1]-15):dim(x)[1]),] # trim off last 16 lines of disclaimer
+x.dates = paste(substring(x[,1],1,6),substring(x[,1],7,10)) # unmangle the dates
+x.dates = as.Date(x.dates, format="%b %d %Y")
+x.prices = as.numeric((sub(",","", x[,2], fixed=TRUE))) # get rid of commas
+x.xts = xts(x.prices, order.by=x.dates)
+x.returns = Return.calculate(x.xts)
+x.xts = cbind(x.xts, x.returns)
+colnames(x.xts) = c("Close","Returns")
+
+# Save it into an rda file on the filesystem
+save(x.xts, file=paste(filesroot,"MSCI.World.IDX/MSCI.World.IDX.rda", sep="/"))
+
+# Create currencies first:
+require(FinancialInstrument)
+currency("USD")
+
+# Describe the metadata for the index
+instrument("MSCI.World.IDX", currency="USD", multiplier=1, tick_size=.01, start_date="1969-12-31", description="MSCI World Index Standard (Large+Mid Cap) USD", data="CR", source="msci.com", assign_i=TRUE)
+
+# Now, whenever you log in you need to register the instruments. This
+# might be a line you put into .Rprofile so that it happens automatically:
+# require(quantmod) # this requires a development build after revision 560 or so.
+setSymbolLookup.FI(base_dir=filesroot, split_method='common')
+
+# Now you should be able to:
+getSymbols("MSCI.World.IDX")
+chartSeries(Cl(MSCI.World.IDX), theme="white")
+head(MSCI.World.IDX)
\ No newline at end of file
Property changes on: pkg/FinancialInstrument/sandbox/parse.MSCI.R
___________________________________________________________________
Added: svn:keywords
+ Revision Id Date Author
Added: pkg/FinancialInstrument/sandbox/parse.SP500TR.R
===================================================================
--- pkg/FinancialInstrument/sandbox/parse.SP500TR.R (rev 0)
+++ pkg/FinancialInstrument/sandbox/parse.SP500TR.R 2011-06-02 02:11:08 UTC (rev 610)
@@ -0,0 +1,61 @@
+# Script for downloading and parsing a monthly total return series from
+# http://www.standardandpoors.com/
+#
+
+# Peter Carl
+
+# Load needed packages:
+require(zoo)
+require(gdata)
+
+# @TODO: Set the working directory
+#
+
+# Download the first sheet in the xls workbook directly from the web site:
+# x = read.xls("http://www2.standardandpoors.com/spf/xls/index/MONTHLY.xls")
+x = read.xls("http://www.standardandpoors.com/servlet/BlobServer?blobheadername3=MDT-Type&blobcol=urldata&blobtable=MungoBlobs&blobheadervalue2=inline%3B+filename%3DMONTHLY.xls&blobheadername2=Content-Disposition&blobheadervalue1=application%2Fexcel&blobkey=id&blobheadername1=content-type&blobwhere=1243862365806&blobheadervalue3=UTF-8")
+
+# That gives us something like the following:
+# > head(x)
+# STANDARD...POOR.S.INDEX.SERVICES X X.1 X.2 X.3 X.4
+# 1 S&P 500 MONTHLY RETURNS
+# 2
+# 3 MONTH OF PRICE PRICE 1 MONTH 3 MONTH 6 MONTH
+# 4 CLOSE CHANGE % CHANGE % CHANGE % CHANGE
+# 5 10/2009 1036.19 -20.88 -1.98% 4.93% 18.72%
+# 6 09/2009 1057.08 36.45 3.57% 14.98% 32.49%
+# X.5 X.6 X.7 X.8 X.9 X.10 X.11 X.12 X.13
+# 1 NA NA
+# 2 1 MONTH 12 MONTH NA NA
+# 3 1 YEAR 2 YEAR 3 YEAR 5 YEARS 10 YEARS TOTAL TOTAL NA NA
+# 4 % CHANGE % CHANGE % CHANGE % CHANGE % CHANGE RETURN RETURN NA NA
+# 5 6.96% -33.12% -24.80% -8.32% -23.97% -1.86% 9.80% NA NA
+# 6 -9.37% -30.76% -20.87% -5.16% -17.59% 3.73% -6.91% NA NA
+# X.14 X.15
+# 1 NA NA
+# 2 NA NA
+# 3 NA NA
+# 4 NA NA
+# 5 NA NA
+# 6 NA NA
+
+# So we only really care about column 1 for dates and column 12 (X.10) for
+# total returns. The first four rows are headers, and can be discarded.
+rawdates = x[-1:-4,1]
+rawreturns = x[-1:-4,12]
+# Data goes back to 12/1988.
+
+# First we convert the dates to something we can use. Note that frac=1 sets
+# the day to the last day of the month. That should be close enough for
+# monthly data.
+ISOdates = as.Date(as.yearmon(rawdates, "%m/%Y"), frac=1)
+
+# Now we convert the rawreturns strings into numbers
+totalreturns = as.numeric(as.character((sub("%", "", rawreturns, fixed=TRUE))))/100
+
+# Now construct an xts object with the two columns
+SP500.TR=na.omit(as.xts(totalreturns, order.by=ISOdates))
+colnames(SP500.TR)="SP500TR"
+
+# @TODO: Save it as rda and register it with FI
+# save(SP500.TR)
Property changes on: pkg/FinancialInstrument/sandbox/parse.SP500TR.R
___________________________________________________________________
Added: svn:keywords
+ Revision Id Date Author
More information about the Blotter-commits
mailing list