Commit 7bb7485a authored by Franssen, Wietse's avatar Franssen, Wietse
Browse files

added regrid and many other changes

parent 1c4a0569
......@@ -17,3 +17,4 @@ testData
*.log
tmpScripts
*.RData
DATA/
\ No newline at end of file
# If using old data
## Step 1
reformat the old 0.75 degree data (seperate years, seperate members, combined leadmonths) to new data (combined years, combined members, seperate leadmonths)
## Step 2
regrid the 0.75 degree data into 0.50 degree data
both datasets are: combined years, combined members, seperate leadmonths
# If downloading new data
## Step 1
......@@ -11,9 +11,12 @@ if (submitscript) {
currMonths <- c(X:X)
leadMonths<-c(0:6)
locName <- 'X'
inPath<-"./testData/reFormatted_noBC"
outPath<-"./testData/reFormatted_BC"
resolution<-"0.50"
# inPath<-"./testData/reFormatted_noBC"
obsPath<-sprintf("../DATA/wfdei_rev3.0/%sdeg/%s/", resolution, locName)
inPath<-sprintf("../DATA/System4_seasonal_15_rev2.0/%sdeg/%s_noBC", resolution, locName)
outPath<-sprintf("../DATA/System4_seasonal_15_rev2.0/%sdeg/%s_BC", resolution, locName)
# outPath<-"./testData/reFormatted_BC"
} else {
members <-c(1:15)
initYears <-c(1981:2010)
......@@ -21,6 +24,7 @@ if (submitscript) {
leadMonths<-c(0:6)
#locName<-"GHA"
locName<-"EU"
obsPath<-sprintf("../DATA/wfdei_rev3.0/%sdeg/%s/", resolution, locName)
inPath<-"./testData/reFormatted_noBC"
outPath<-"./testData/reFormatted_BC"
resolution<-"0.50"
......@@ -29,6 +33,7 @@ if (submitscript) {
variables<-names(variableInfo)
variables<-c( "rsds" , "rlds" , "tasmin","tasmax", "huss" , "sfcWind","pr" , "psl")
variables<-c( "tasmin","tasmax", "huss" , "sfcWind","pr" , "psl")
variables<-c( "pr", "sfcWind", "rsds", "rlds", "huss", "tas", "tasmin", "tasmax")
#variables<-c("tas")
dir.create(outPath, recursive = TRUE, showWarnings = FALSE)
......@@ -48,25 +53,31 @@ for (variableName in variables) {
RData_noBC<-RData
rm(RData)
obsFile <- sprintf("/home/WUR/frans004/L_BACKUP/PROJECTS/EUPORIAS/DATA_BIAS/DATA/DATA_RAW_R/%s_WFD/%s_wfdei_%4d-%4d_%02d.RData",
locName, variableName,
# obsFile <- sprintf("/home/WUR/frans004/L_BACKUP/PROJECTS/EUPORIAS/DATA_BIAS/DATA/DATA_RAW_R/%s_WFD/%s_wfdei_%4d-%4d_%02d.RData",
# locName, variableName,
# initYears[1], initYears[length(initYears)],
# currMonth)
obsFile <- sprintf("%s/%s_wfdei_%s_%4d-%4d_%02d.RData",
obsPath, variableName, locName,
initYears[1], initYears[length(initYears)],
currMonth)
print(sprintf("Open: %s", obsFile))
load(obsFile)
obs_new<-obs
newGrid<-getGrid(RData_noBC)
indexesLat<-c(which(obs$xyCoords$y==newGrid$y[1]):which(obs$xyCoords$y==newGrid$y[2]))
indexesLon<-c(which(obs$xyCoords$x==newGrid$x[1]):which(obs$xyCoords$x==newGrid$x[2]))
obs_new$xyCoords<-RData_noBC$xyCoords
obs_new$Data<-obs$Data[,indexesLat,indexesLon]
attr(obs_new$Data,"dimensions") <- c("time","lat","lon")
obs<-obs_new
rm(obs_new)
RData_obs<-RData
rm(RData)
# newGrid<-getGrid(RData_noBC)
# indexesLat<-c(which(RData$xyCoords$y==newGrid$y[1]):which(RData$xyCoords$y==newGrid$y[2]))
# indexesLon<-c(which(RData$xyCoords$x==newGrid$x[1]):which(RData$xyCoords$x==newGrid$x[2]))
# RData_new$xyCoords<-RData_noBC$xyCoords
# RData_new$Data<-RData$Data[,indexesLat,indexesLon]
# attr(RData_new$Data,"dimensions") <- c("time","lat","lon")
# RData<-RData_new
# rm(RData_new)
print(sprintf("Bais correction..."))
RData <- biasCorrection (obs, RData_noBC, RData_noBC, method = "qqmap", multi.member = FALSE, pr.threshold = 0.1)# bias
RData <- biasCorrection (RData_obs, RData_noBC, RData_noBC, method = "qqmap", multi.member = FALSE, pr.threshold = 0.1)# bias
oFile <- sprintf("%s/%s_forcing_seas15_%s_BC_E%02d-%02d_%4d-%4d_%02d_LM%d.RData",
outPath, variableName, locName,
......
......@@ -17,11 +17,11 @@ if (submitscript) {
targetMonths <- c(1:12)
# targetMonths <- c(1)
targetYears <- c(1981:2011)
targetYears <- c(1981:1982)
# targetYears <- c(1981:1982)
leadMonths <- c(X:X)
locName <- 'X'
# outPath<-"../DATA/System4_seasonal_15/0.75deg/EU_noBC"
outPath<-sprintf("../DATA/System4_seasonal_15/0.75deg/%s_noBC", locName)
outPath<-sprintf("../DATA/System4_seasonal_15_rev3.0/0.75deg/%s_noBC", locName)
} else {
members <-c(1,2)
targetMonths <-c(1:2)
......@@ -35,9 +35,9 @@ if (submitscript) {
}
variables<-names(variableInfo)
variables<-c("pr", "rsds", "rlds")
variables<-c( "sfcWind", "psl" , "huss", "tas", "tasmin", "tasmax")
variables<-c("tas", "rsds", "rlds", "pr")
#variables<-c("pr", "rsds", "rlds")
#variables<-c( "sfcWind", "psl" , "huss", "tas", "tasmin", "tasmax")
#variables<-c("tas", "rsds", "rlds", "pr")
dir.create(outPath, recursive = TRUE, showWarnings = FALSE)
......
rm(list=ls())
library(fields) # e.g: using the fields library
library(abind)
library(ncdf4)
library(ecomsUDG.Raccess)
library(ecomsUDG.Raccess)
source(file = "./functions/infoGeneral.R")
source(file = "./functions/functionsGeneral.R")
source(file = "./functions/functionR2Netcdf.R")
loginECOMS_UDG(username = "wietsefranssen", password = "ECOMS")
submitscript <- FALSE
if (submitscript) {
targetMonths <- c(1:12)
targetYears <- c(1981:2011)
locName <- 'X'
resolution<-"0.50"
outPath<-sprintf("../DATA/wfdei_rev3.0/%sdeg/%s", resolution, locName)
} else {
targetMonths <-c(1:12)
targetYears <-c(1981:2010)
# targetYears <-c(1981:1982)
locName<-"GHA"
locName<-"EU"
resolution<-"0.75"
outPath<-sprintf("../DATA/wfdei_rev3.0/%sdeg/%s", resolution, locName)
}
variables<-names(variableInfo)
variables<-c( "pr", "sfcWind", "rsds", "rlds", "huss", "tas", "tasmin", "tasmax")
dir.create(outPath, recursive = TRUE, showWarnings = FALSE)
for (variableName in variables) {
for (targetMonth in targetMonths) {
if ('ecomsName' %in% names(variableInfo[[variableName]])) {
variableNameECOMS<-variableInfo[[variableName]]$ecomsName
} else {
variableNameECOMS<-variableName
}
print(sprintf("targetMonth: %s, years: %d-%d",month.name[targetMonth], targetYears[1], targetYears[length(targetYears)]))
oPrefix <- sprintf("%s/%s_wfdei_%s_%4d-%4d_%02d",
outPath, variableName, locName,
targetYears[1], targetYears[length(targetYears)],
targetMonth)
print(oPrefix)
if (resolution == "0.50") {
lonLim = c(locationInfo$res0.50[[locName]]$lonmin, locationInfo$res0.50[[locName]]$lonmax)
latLim = c(locationInfo$res0.50[[locName]]$latmin, locationInfo$res0.50[[locName]]$latmax)
} else {
# lonLim = c(locationInfo$res0.75[[locName]]$lonmin-1.5, locationInfo$res0.75[[locName]]$lonmax+1.5)
# latLim = c(locationInfo$res0.75[[locName]]$latmin-1.5, locationInfo$res0.75[[locName]]$latmax+1.5)
lonLim = c(locationInfo$res0.75[[locName]]$lonmin, locationInfo$res0.75[[locName]]$lonmax)
latLim = c(locationInfo$res0.75[[locName]]$latmin, locationInfo$res0.75[[locName]]$latmax)
}
RData<-loadECOMS(dataset = "WFDEI",
var = variableNameECOMS,
lonLim = lonLim,
latLim = latLim,
season = targetMonth,
years = targetYears,
time = "DD")
RData$xyCoords$x[]<-round(RData$xyCoords$x[],2)
RData$xyCoords$y[]<-round(RData$xyCoords$y[],2)
if (resolution == "0.75") {
newGrid<-getGrid(RData)
newGrid$x <- c(locationInfo$res0.75[[locName]]$lonmin, locationInfo$res0.75[[locName]]$lonmax)
newGrid$y <- c(locationInfo$res0.75[[locName]]$latmin, locationInfo$res0.75[[locName]]$latmax)
attr(newGrid,"resX")<-0.75
attr(newGrid,"resY")<-0.75
print("Regridding")
RData <- interpGridData(RData, new.grid = newGrid, method = "bilinear");
}
# Check units
RData$Variable$varName<-variableName
attr(RData$Variable,"standard_name")<-variableInfo[[variableName]]$standardName
attr(RData$Variable,"long_name")<-variableInfo[[variableName]]$longName
attr(RData$Variable,"units")<-variableInfo[[variableName]]$unitsEcoms
## add some extra attributes
attr(RData,"contact") <- "Wietse Franssen (wietse.franssen@wur.nl)"
save(file = sprintf("%s.RData",oPrefix), RData)
# R2Netcdf(sprintf("%s.nc4",oPrefix), RData)
}
}
......@@ -3,10 +3,10 @@ library(fields) # e.g: using the fields library
library(abind)
library(ncdf4)
library(ecomsUDG.Raccess)
source(file = "./functionReformat.R")
source(file = "./functionsGeneral.R")
source(file = "./functionR2Netcdf.R")
source(file = "./infoGeneral.R")
source(file = "./functions/functionReformat.R")
source(file = "./functions/functionsGeneral.R")
source(file = "./functions/functionR2Netcdf.R")
source(file = "./functions/infoGeneral.R")
submitscript <- FALSE
......@@ -15,65 +15,43 @@ if (submitscript) {
initYears <-c(1981:2010)
initMonths <- c(X:X)
locName <- 'X'
inPath <- "/home/WUR/frans004/L_BACKUP/PROJECTS/EUPORIAS/DATA_BIAS/DATA/DATA_RAW_R/EU_0.50/"
outPath<-"./testData/reFormatted"
resolution<-"0.75"
inPath<-sprintf("../DATA/System4_seasonal_15_rev1.0/%s_%s", locName, resolution)
outPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_noBC", resolution, locName)
} else {
members <-c(1:15)
members <-c(1:2)
initYears <-c(1981:1982)
initMonths<-1
#locName<-"GHA"
locName<-"EU"
inPath <- "./testData/RAW-test/"
outPath<-"./testData/reFormatted"
resolution<-"0.75"
inPath<-sprintf("../DATA/System4_seasonal_15_rev1.0/%s_%s", locName, resolution)
outPath<-sprintf("../DATA/System4_seasonal_15_rev1.x/%sdeg/%s_noBC", resolution, locName)
}
variables<-names(variableInfo)
variables<-c( "pr" , "psl" , "rsds" , "rlds" , "tasmin","tasmax", "huss" , "sfcWind")
variables<-c( "tp" , "psl" , "rsds" , "rlds" , "tasmin","tasmax", "huss" , "sfcWind")
variables<-c( "wss")
#variables<-c("tas")
#variables<-c( "pr" , "psl" , "rsds" , "rlds" , "tasmin","tasmax", "huss" , "sfcWind")
#variables<-c( "pr", "sfcWind")
dir.create(outPath, recursive = TRUE, showWarnings = FALSE)
#targetMonth<-1
leadMonths<-c(0:6)
#initInfo <- getInitInfo(targetYear = 1981, targetMonth = 6, leadMonth = 0)
#availableTargetYears <- getAvailableTargetYears(targetYears = targetYears, targetMonth = targetMonth, leadMonth = 0)
#print(initInfo)
print("start")
for (initMonth in initMonths) {
for (variableName in variables) {
# inFile <- sprintf("%s/%s_forcing_seas15_%s_noBC_E<MEMBERS>_<YEARS>_%02d.RData", inPath, variableName, locName, initMonth)
inFile <- sprintf("%s/%s/%s_forcing_seas15_%s_noBC_E<MEMBERS>_<YEARS>_%02d.RData", inPath, variableName, variableName, locName, initMonth)
print(inFile)
#RData <- reformat2Bias(members, initYears, initInfo$initMonth, varName, locName, iFile = inFile)
RDataAllLeadMonths <- reformat2Bias(members, initYears, initMonth, variableName, locName, iFile = inFile)
for (leadMonth in leadMonths) {
currMonth<-indexesOfDaysPerMonth(initMonth,7,0000)[leadMonth+1,"month"]
print(sprintf("currmonth: %s, leadMonth: %s",currMonth,leadMonth))
## get the available target years for the current targetMonth and leadMonth
# availableTargetYears <- getAvailableTargetYears(targetYears = targetYears, targetMonth = targetMonth, leadMonth = leadMonth)
# nYears <- length(availableTargetYears)
#
if ('ecomsName' %in% names(variableInfo[[variableName]])) {
variableNameECOMS<-variableInfo[[variableName]]$ecomsName
} else {
variableNameECOMS<-variableName
}
inFile <- sprintf("%s/%s/%s_forcing_seas15_%s_noBC_E<MEMBERS>_<YEARS>_%02d.RData", inPath, variableNameECOMS, variableNameECOMS, locName, initMonth)
print(inFile)
RDataAllLeadMonths <- reformat2Bias(members, initYears, initMonth, variableNameECOMS, locName, iFile = inFile)
# print(sprintf("targetMonth: %s, leadMonth: %s, years: %d-%d",month.name[targetMonth], leadMonth, availableTargetYears[1], availableTargetYears[nYears]))
for (leadMonth in leadMonths) {
currMonth<-indexesOfDaysPerMonth(initMonth,7,0000)[leadMonth+1,"month"]
print(sprintf("currmonth: %s, leadMonth: %s",currMonth, leadMonth))
# oPrefix <- sprintf("%s/%s_forcing_seas15_%s_noBC_E%02d-%02d_%4d-%4d_%02d_LM%d",
# outPath, variableName, locName,
# members[1], members[length(members)],
# availableTargetYears[1], availableTargetYears[nYears],
# targetMonth, leadMonth)
oPrefix <- sprintf("%s/%s_forcing_seas15_%s_noBC_E%02d-%02d_%4d-%4d_%02d_LM%d",
outPath, variableName, locName,
members[1], members[length(members)],
......@@ -99,13 +77,10 @@ for (initMonth in initMonths) {
## add some extra attributes
attr(RData,"contact") <- "Wietse Franssen (wietse.franssen@wur.nl)"
#print(names(RDataAllLeadMonths))
# print(names(RData))
save(RData, file = paste0(oPrefix, ".RData"))
R2Netcdf(paste0(oPrefix, ".nc4"), RData)
#R2Netcdf(paste0(oPrefix, ".nc4"), RData)
}
}
}
\ No newline at end of file
rm(list=ls())
library(ecomsUDG.Raccess)
source(file = "./functions/infoGeneral.R")
submitscript <- FALSE
if (submitscript) {
members <- c(1:15)
initYears <-c(1981:2010)
currMonths <- c(X:X)
leadMonths<-c(0:6)
locName <- 'X'
fromResolution <- "0.75"
toResolution <- "0.50"
inPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_noBC", fromResolution, locName)
outPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_noBC", toResolution, locName)
} else {
members <-c(1:15)
initYears <-c(1981:2010)
currMonths<-1
leadMonths<-c(0:6)
#locName<-"GHA"
locName<-"EU"
fromResolution <- "0.75"
toResolution <- "0.50"
inPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_noBC", fromResolution, locName)
outPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_noBC", toResolution, locName)
}
variables<-names(variableInfo)
variables<-c("tas")
dir.create(outPath, recursive = TRUE, showWarnings = FALSE)
print("start")
for (variableName in variables) {
for (currMonth in currMonths) {
for (leadMonth in leadMonths) {
iFile <- sprintf("%s/%s_forcing_seas15_%s_noBC_E%02d-%02d_%4d-%4d_%02d_LM%d.RData",
inPath, variableName, locName,
members[1], members[length(members)],
initYears[1], initYears[length(initYears)],
currMonth, leadMonth)
print(sprintf("Open: %s", iFile))
load(iFile)
newGrid<-getGrid(RData)
newGrid$x <- c(locationInfo$res0.50[[locName]]$lonmin, locationInfo$res0.50[[locName]]$lonmax)
newGrid$y <- c(locationInfo$res0.50[[locName]]$latmin, locationInfo$res0.50[[locName]]$latmax)
attr(newGrid,"resX") <- 0.50
attr(newGrid,"resY") <- 0.50
print(sprintf("Regridding..."))
RData <- interpGridData(RData, new.grid = newGrid, method = "bilinear");
oFile <- sprintf("%s/%s_forcing_seas15_%s_BC_E%02d-%02d_%4d-%4d_%02d_LM%d.RData",
outPath, variableName, locName,
members[1], members[length(members)],
initYears[1], initYears[length(initYears)],
currMonth, leadMonth)
print(sprintf("Saving: %s", oFile))
save(file= oFile, RData)
}
}
}
......@@ -17,6 +17,6 @@ module load R/3.2.0
#
# run current job
echo '******** R-data Download...'
Rscript ./tmpScripts/biascorrection_<location>_currMonth<currMonth>.R
Rscript ./tmpScripts/doBiascorrection_<location>_currMonth<currMonth>.R
date
#!/bin/bash
#SBATCH --account=5120867-01
#SBATCH --time=5000
#SBATCH --time=50000
#SBATCH --mem=16024
#SBATCH --ntasks=1
#SBATCH --output=./log/log_download_<location>_leadMonth<leadMonth>_%j.txt
......
......@@ -16,7 +16,7 @@ pwd
module load R/3.2.0
#
# run current job
echo '******** R-data Download...'
Rscript ./tmpScripts/reformat_<location>_initMonth<initMonth>.R
echo '******** start Rscript...'
Rscript ./tmpScripts/doReformat_<location>_initMonth<initMonth>.R
date
#!/bin/bash
#SBATCH --account=5120867-01
#SBATCH --time=5000
#SBATCH --mem=16024
#SBATCH --ntasks=1
#SBATCH --output=./log/log_regrid_<location>_currMonth<currMonth>_%j.txt
#SBATCH --job-name=regr
#SBATCH --partition=ESG_Std
#SBATCH --mail-type=FAIL
#SBATCH --mail-user=wietse.franssen@wur.nl
# print date and time
date
#
# print the working directory
pwd
module load R/3.2.0
#
# run current job
echo '******** start Rscript...'
Rscript ./tmpScripts/doRegrid_<location>_currMonth<currMonth>.R
date
......@@ -4,7 +4,7 @@ tmpScripts="./tmpScripts"
location="EU"
mkdir -p $tmpScripts
for iCurrMonth in $(seq -f "%1g" 1 12); do
for iCurrMonth in $(seq -f "%1g" 1 1); do
echo "leadMonth: $iCurrMonth"
sed -e "s|<currMonth>|$iCurrMonth|g" \
-e "s|<location>|$location|g" \
......@@ -12,6 +12,6 @@ for iCurrMonth in $(seq -f "%1g" 1 12); do
sed -e "s|submitscript <- FALSE|submitscript <- TRUE|g" \
-e "s|currMonths <- c(X:X)|currMonths <- c($iCurrMonth:$iCurrMonth)|g" \
-e "s|locName <- 'X'|locName <- '$location'|g" \
biascorrection.R > $tmpScripts"/biascorrection_"$location"_currMonth"$iCurrMonth".R"
doBiascorrection.R > $tmpScripts"/doBiascorrection_"$location"_currMonth"$iCurrMonth".R"
sbatch "$tmpScripts/jobScriptBiascorrection_"$location"_currMonth"$iCurrMonth
done
......@@ -12,6 +12,6 @@ for iInitMonth in $(seq -f "%1g" 1 12); do
sed -e "s|submitscript <- FALSE|submitscript <- TRUE|g" \
-e "s|initMonths <- c(X:X)|initMonths <- c($iInitMonth:$iInitMonth)|g" \
-e "s|locName <- 'X'|locName <- '$location'|g" \
reformat.R > $tmpScripts"/reformat_"$location"_initMonth"$iInitMonth".R"
doReformat.R > $tmpScripts"/doReformat_"$location"_initMonth"$iInitMonth".R"
sbatch "$tmpScripts/jobScriptReformat_"$location"_initMonth"$iInitMonth
done
#!/bin/bash
tmpScripts="./tmpScripts"
#location="GHA"
location="EU"
mkdir -p $tmpScripts
for iCurrMonth in $(seq -f "%1g" 1 12); do
echo "leadMonth: $iCurrMonth"
sed -e "s|<currMonth>|$iCurrMonth|g" \
-e "s|<location>|$location|g" \
jobScriptRegrid > $tmpScripts"/jobScriptRegrid_"$location"_currMonth"$iCurrMonth
sed -e "s|submitscript <- FALSE|submitscript <- TRUE|g" \
-e "s|currMonths <- c(X:X)|currMonths <- c($iCurrMonth:$iCurrMonth)|g" \
-e "s|locName <- 'X'|locName <- '$location'|g" \
doRegrid.R > $tmpScripts"/doRegrid_"$location"_currMonth"$iCurrMonth".R"
sbatch "$tmpScripts/jobScriptRegrid_"$location"_currMonth"$iCurrMonth
done
......@@ -53,7 +53,6 @@ for (variableName in variables) {
variableNameECOMS<-variableName
}
print(sprintf("targetMonth: %s, leadMonth: %s, years: %d-%d",month.name[targetMonth], leadMonth, availableTargetYears[1], availableTargetYears[nYears]))
oPrefix <- sprintf("%s/%s_forcing_seas15_%s_noBC_E%02d-%02d_%4d-%4d_%02d_LM%d",
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment