Talk:CLAIH assessment

Temperature data

• To load the data into R use the following for the whole data.

Data <- read.csv("N:/YMTO/PROJECTS/CLAIH/CLAIH data/tmax2000.csv", dec = ",", sep = ";")

• And the following for the Europe slice.

Data <- read.csv("N:/YMTO/PROJECTS/CLAIH/CLAIH data/tmax2000sliced.csv", dec = ",", sep = ";")

• To open 'ff' objects generated by the codes below, use ffload("filepath/filename with no file extension"). ff is an additional package for R, that makes it possible to use on disk objects; this enables usage of huge data objects without memory problems.

• The R Code below reads the original .txt file and saves it as a 'ff' file as well as .csv, also produces a slice of the data with only Europe in it.
  • ff, an R package that handles large data as on disk objects, is used in the code. It can easily be installed from within R.

library(ff)
Data <- read.table("N:/YMTO/PROJECTS/CLAIH/CLAIH data/tmax2000.txt", fill=TRUE)
library(ff)
Year <- ff(as.factor(Data[(1:(nrow(Data)/2)*2-1),1]), filename = "M:/ff/temperature/Year", vmode = "quad") #2 bit unsigned integer, 
#enough for only one factor level ("2000")
Month <- ff(as.factor(Data[(1:(nrow(Data)/2)*2-1),2]), filename = "M:/ff/temperature/Month", vmode = "nibble") #4 bit unsigned integer, 
#2^4 = 16 combinations, enough for months
Day <- ff(as.factor(Data[(1:(nrow(Data)/2)*2-1),3]), filename = "M:/ff/temperature/Day", vmode = "ubyte") #8 bit unsigned integer
Latitude <- ff(Data[(1:(nrow(Data)/2)*2-1),4], filename = "M:/ff/temperature/Latitude", vmode="single")
Longitude <- ff(Data[(1:(nrow(Data)/2)*2-1),5], filename = "M:/ff/temperature/Longitude", vmode="single")
Temperature <- ff(Data[(1:(nrow(Data)/2)*2),1], filename = "M:/ff/temperature/Temperature")
remove(Data)
Data <- ffdf(Year, Month, Day, Latitude, Longitude, Temperature)
ffsave(Data, file = "M:/ff/temperature") #ff object save
test <- Data[Data[,"Latitude"]>=30,] #data slicing
test <- test[test[,"Latitude"]<=75,] #data slicing
test <- test[test[,"Longitude"]>=330|test[,"Longitude"]<=45,] #data slicing
test <- as.ffdf(test)
ffsave(test, file = "M:/ff/temperature.sliced") #ff object save
write.table.ffdf(Data, "N:/YMTO/PROJECTS/CLAIH/CLAIH data/tmax2000.csv", row.names = FALSE, sep = ";")#, dec = ",") #whole data .csv save
write.table.ffdf(test, "N:/YMTO/PROJECTS/CLAIH/CLAIH data/tmax2000sliced.csv", row.names = FALSE, sep = ";")#, dec = ",") #sliced data .csv save

• The code below reads the .csv file produced by code above and saves it as an 'ff' file.

library(ff)
Data <- read.csv.ffdf("N:/YMTO/PROJECTS/CLAIH/CLAIH data/tmax2000.csv", dec = ",", sep = ";")
Year <- ff(as.factor(Data[,1]), filename = "M:/ff/temperature/Year", overwrite = TRUE, vmode = "quad") #2 bit unsigned, enough for only one 
#factor level ("2000")
Month <- ff(as.factor(Data[,2]), filename = "M:/ff/temperature/Month", overwrite = TRUE, vmode = "nibble") #4 bit unsigned integer, 2^4 = 16 
#combinations, enough for months
Day <- ff(as.factor(Data[,3]), filename = "M:/ff/temperature/Day", overwrite = TRUE, vmode = "ubyte") #8 bit unsigned integer
Latitude <- ff(Data[,4], filename = "M:/ff/temperature/Latitude", overwrite = TRUE)
Longitude <- ff(Data[,5], filename = "M:/ff/temperature/Longitude", overwrite = TRUE)
Temperature <- ff(Data[,6], filename = "M:/ff/temperature/Temperature", overwrite = TRUE)
remove(Data)
Data <- ffdf(Year, Month, Day, Latitude, Longitude, Temperature)
ffsave(Data, file = "M:/ff/temperature")
test <- Data[Data[,"Latitude"]>=30,]
test <- test[test[,"Latitude"]<=75,]
test <- test[test[,"Longitude"]>=330|test[,"Longitude"]<=45,]
test <- as.ffdf(test)
ffsave(test, file = "M:/ff/temperature.sliced")