Skip to content
GitLab
Commit b7a9c42b authored by Franssen, Wietse's avatar Franssen, Wietse
Browse files

Tydied up

parent 3158c831
Hide whitespace changes
Inline Side-by-side
SCRIPTS/3_submit_reorder_combYears_combMembers_sepLeadmonths.sh 0 → 100755
View file @ b7a9c42b
#!/bin/bash
tmpScripts="./tmpScripts"
#location="EU"; resolution="0.50"
#location="GHA"; resolution="0.50"
location="GHA"; resolution="0.75"
mkdir -p $tmpScripts
for iInitMonth in $(seq -f "%02g" 1 12); do
echo "leadMonth: $iInitMonth"
sed -e "s|<initMonth>|$iInitMonth|g" \
-e "s|<location>|$location|g" \
-e "s|<resolution>|$resolution|g" \
jobScriptReorder_combYears_combMembers_sepLeadmonths > $tmpScripts"/jobScriptReorder_combYears_combMembers_sepLeadmonths_"$location"_"$resolution"deg_initMonth"$iInitMonth
sed -e "s|submitscript <- FALSE|submitscript <- TRUE|g" \
-e "s|initMonths <- c(X:X)|initMonths <- c($iInitMonth:$iInitMonth)|g" \
-e "s|locName <- 'X'|locName <- '$location'|g" \
-e "s|resolution <- 'X'|resolution <- '$resolution'|g" \
doReorder_combYears_combMembers_sepLeadmonths.R > $tmpScripts"/doReorder_combYears_combMembers_sepLeadmonths_"$location"_"$resolution"deg_initMonth"$iInitMonth".R"
sbatch "$tmpScripts/jobScriptReorder_combYears_combMembers_sepLeadmonths_"$location"_"$resolution"deg_initMonth"$iInitMonth
done
SCRIPTS/submit_biascorrection.sh → SCRIPTS/4_submit_biascorrection.sh
View file @ b7a9c42b
#!/bin/bash
tmpScripts="./tmpScripts"
#location="GHA"
location="EU"
#location="EU"; resolution="0.50"
#location="GHA"; resolution="0.50"
location="GHA"; resolution="0.75"
mkdir -p $tmpScripts
for iTargetMonth in $(seq -f "%02g" 1 12); do
echo "leadMonth: $iTargetMonth"
sed -e "s|<targetMonth>|$iTargetMonth|g" \
-e "s|<location>|$location|g" \
jobScriptBiascorrection > $tmpScripts"/jobScriptBiascorrection_"$location"_targetMonth"$iTargetMonth
-e "s|<resolution>|$resolution|g" \
jobScriptBiascorrection > $tmpScripts"/jobScriptBiascorrection_"$location"_"$resolution"deg_targetMonth"$iTargetMonth
sed -e "s|submitscript <- FALSE|submitscript <- TRUE|g" \
-e "s|targetMonths <- c(X:X)|targetMonths <- c($iTargetMonth:$iTargetMonth)|g" \
-e "s|locName <- 'X'|locName <- '$location'|g" \
doBiascorrection.R > $tmpScripts"/doBiascorrection_"$location"_targetMonth"$iTargetMonth".R"
sbatch "$tmpScripts/jobScriptBiascorrection_"$location"_targetMonth"$iTargetMonth
-e "s|targetMonths <- c(X:X)|targetMonths <- c($iTargetMonth:$iTargetMonth)|g" \
-e "s|locName <- 'X'|locName <- '$location'|g" \
-e "s|resolution <- 'X'|resolution <- '$resolution'|g" \
doBiascorrection.R > $tmpScripts"/doBiascorrection_"$location"_"$resolution"deg_targetMonth"$iTargetMonth".R"
sbatch "$tmpScripts/jobScriptBiascorrection_"$location"_"$resolution"deg_targetMonth"$iTargetMonth
done
SCRIPTS/submit_reorder_combYears_combMembers_sepLeadmonths.sh → SCRIPTS/6_submit_netcdfCombine.sh
View file @ b7a9c42b
......@@ -8,10 +8,10 @@ for iInitMonth in $(seq -f "%02g" 1 12); do
echo "leadMonth: $iInitMonth"
sed -e "s|<initMonth>|$iInitMonth|g" \
-e "s|<location>|$location|g" \
jobScriptReorder_combYears_combMembers_sepLeadmonths > $tmpScripts"/jobScriptReorder_combYears_combMembers_sepLeadmonths_"$location"_initMonth"$iInitMonth
jobScriptNetcdfCombine > $tmpScripts"/jobScriptNetcdfCombine_"$location"_initMonth"$iInitMonth
sed -e "s|submitscript <- FALSE|submitscript <- TRUE|g" \
-e "s|initMonths <- c(X:X)|initMonths <- c($iInitMonth:$iInitMonth)|g" \
-e "s|locName <- 'X'|locName <- '$location'|g" \
doReorder_combYears_combMembers_sepLeadmonths.R > $tmpScripts"/doReorder_combYears_combMembers_sepLeadmonths_"$location"_initMonth"$iInitMonth".R"
sbatch "$tmpScripts/jobScriptReorder_combYears_combMembers_sepLeadmonths_"$location"_initMonth"$iInitMonth
doNetcdfCombine.R > $tmpScripts"/doNetcdfCombine_"$location"_initMonth"$iInitMonth".R"
sbatch "$tmpScripts/jobScriptNetcdfCombine_"$location"_initMonth"$iInitMonth
done
SCRIPTS/doBiascorrection.R
View file @ b7a9c42b
......@@ -12,33 +12,31 @@ source(file = "./functions/functionR2Netcdf.R")
submitscript <- FALSE
if (submitscript) {
members <- c(1:15)
initYears <-c(1981:2010)
targetMonths <- c(X:X)
leadMonths<-c(0:6)
locName <- 'X'
resolution<-"0.50"
obsPath<-sprintf("../DATA/wfdei_rev3.0/%sdeg/%s/", resolution, locName)
inPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_noBC", resolution, locName)
outPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_BC", resolution, locName)
members <- c(1:15)
initYears <- c(1981:2010)
targetMonths <- c(X:X)
leadMonths <- c(0:6)
locName <- 'X'
resolution <- "0.50"
obsPath <- sprintf("../DATA/wfdei_rev3.0/%sdeg/%s/", resolution, locName)
inPath <- sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_noBC", resolution, locName)
outPath <- sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_BC", resolution, locName)
} else {
members <-c(1:15)
initYears <-c(1981:2010)
targetMonths<-6
leadMonths<-c(0:6)
#locName<-"GHA"
locName<-"EU"
resolution<-"0.50"
obsPath<-sprintf("../DATA/wfdei_rev3.0/%sdeg/%s/", resolution, locName)
inPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_noBC", resolution, locName)
outPath<-sprintf("../DATA/System4_seasonal_15_rev1.2_1.0mm_adapt/%sdeg/%s_BC", resolution, locName)
members <- c(1:15)
initYears <- c(1981:2010)
targetMonths <- 6
leadMonths <- c(0:6)
#locName <- "GHA"
locName <- "EU"
resolution <- "0.50"
obsPath <- sprintf("../DATA/wfdei_rev3.0/%sdeg/%s/", resolution, locName)
inPath <- sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_noBC", resolution, locName)
outPath <- sprintf("../DATA/System4_seasonal_15_rev1.2_1.0mm_adapt/%sdeg/%s_BC", resolution, locName)
}
variables<-names(variableInfo)
variables<-c( "tas", "tasmin", "tasmax", "rsds", "rlds", "pr", "huss", "sfcWind")
variables<-c( "tas", "tasmin", "tasmax", "rsds", "rlds", "huss", "sfcWind")
#variables<-c("pr")
dir.create(outPath, recursive = TRUE, showWarnings = FALSE)
......@@ -118,7 +116,6 @@ for (variableName in variables) {
print(sprintf("Saving: %s", oPrefix))
#save(file = paste0(oPrefix, ".RData"), RData)
R2Netcdf(paste0(oPrefix, ".nc4"), RData)
}
}
}
SCRIPTS/doNetcdfCombine.R 0 → 100644
View file @ b7a9c42b
rm(list=ls())
library(ncdf4)
source(file = "./functions/functionsGeneral.R")
source(file = "./functions/functionReformat.R")
source(file = "./functions/functionR2Netcdf.R")
source(file = "./functions/functionNetcdf2R.R")
source(file = "./functions/functionConvert.R")
source(file = "./functions/infoGeneral.R")
submitscript <- FALSE
if (submitscript) {
members <- c(1:15)
initYears <-c(1981:2010)
initMonths <- c(X:X)
locName <- 'X'
resolution<-"0.50"
# bcInfo<-"BC"
bcInfo<-"noBC"
inPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_%s_modelFormat_perVariable", resolution, locName, bcInfo)
outPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_%s_modelFormat_complete", resolution, locName, bcInfo)
} else {
members <-c(1:15)
initYears <-c(1981:2010)
initMonths<-c(1:12)
#locName<-"GHA"
locName<-"EU"
resolution<-"0.50"
# bcInfo<-"BC"
bcInfo<-"noBC"
inPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_%s_modelFormat_perVariable", resolution, locName, bcInfo)
outPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_%s_modelFormat_complete", resolution, locName, bcInfo)
}
variables<-c( "pr", "psl", "rsds", "rlds", "tas", "tasmin", "tasmax", "huss", "sfcWind")
variables<-c( "pr", "rsds", "rlds", "tas", "tasmin", "tasmax", "huss", "sfcWind")
dir.create(outPath, recursive = TRUE, showWarnings = FALSE)
for (iMember in members) {
for (initYear in initYears) {
for (initMonth in initMonths) {
firstVar<-TRUE
for (variableName in variables) {
## OPEN NCin-Data
nameFileNCin <- sprintf("%s/%s_forcing_seas15_%s_%s_E%02d_%4d_%02d.nc4",
inPath,
variableName,
locName,
bcInfo,
iMember,
initYear,
initMonth)
print(paste0("Reading: ", nameFileNCin))
ncid_in=nc_open(nameFileNCin)
NCdata <- ncvar_get( ncid_in, variableName )
NCtime <- ncvar_get( ncid_in, "time" )
NClon <- ncvar_get( ncid_in, "lon" )
NClat <- ncvar_get( ncid_in, "lat" )
NCtimeAtt<-ncatt_get( ncid_in, "time", "units" )$value
nc_close(ncid_in)
## CREATE NETCDF
if (firstVar == TRUE) {
dimT <- ncdim_def("time", NCtimeAtt, NCtime, unlim = FALSE)
dimY <- ncdim_def("lat", "degrees_north",round(NClat,2))
dimX <- ncdim_def("lon", "degrees_east", round(NClon,2))
FillValue <- 1e20
## SAVE AS NC-DATA
nameFileNCout <- sprintf("%s/forcing_seas15_%s_%s_E%02d_%4d_%02d.nc4",
outPath,
locName,
bcInfo,
iMember,
initYear,
initMonth)
data <- ncvar_def( variableName, variableInfo[[variableName]]$units, list(dimX,dimY,dimT), FillValue, prec="float")#, compression=4)
print(paste0("Creating: ", nameFileNCout, " for: ", variableName))
ncid_out <- nc_create( nameFileNCout,data)
## ADD ATTRIBUTES
ncatt_put( ncid_out, 0, "source1", "distributed by ECOMS User Data Gateway (ECOMS-UDG), partially funded by the European Union FP under Grant Agreement no 308291 (EUPORIAS), http://meteo.unican.es/ecoms-udg")
ncatt_put( ncid_out, 0, "source2", "IPR remains with European Centre For Medium-Range Weather Forecasts (ECMWF)")
ncatt_put( ncid_out, 0, "source3", "ECMWF System4 hind cast, seasonal-15: 12 Runtimes per year (the 1st of J,F,etc) running for 7 months. Ensemble of 15 members for 1981-2010")
ncatt_put( ncid_out, 0, "source4", "variables renamed, domain subset, masking, reorganised, interpolated to 0.5 degree by Wageningen University, Earth System Sciences")
ncatt_put( ncid_out, 0, "source5", "variables naming convention: CMIP5 (http://cmip-pcmdi.llnl.gov/cmip5/docs/standard_output.pdf)")
ncatt_put( ncid_out, 0, "domain", locationInfo[[resolution]][[locName]]$longName)
ncatt_put( ncid_out, 0, "member", iMember)
ncatt_put( ncid_out, 0, "comment1", "Processed and regridded by W.Franssen")
ncatt_put( ncid_out, 0, "institution", "Wageningen University and Research centre (WUR)")
ncatt_put( ncid_out, "lon", "standard_name", "longitude")
ncatt_put( ncid_out, "lon", "long_name", "Longitude")
ncatt_put( ncid_out, "lon", "axis", "X")
ncatt_put( ncid_out, "lat", "standard_name", "latitude")
ncatt_put( ncid_out, "lat", "long_name", "Latitude")
ncatt_put( ncid_out, "lat", "axis", "Y")
ncatt_put( ncid_out, "time", "standard_name", "time")
ncatt_put( ncid_out, "time", "calendar", "standard")
} else {
print(paste0("Adding to: ", nameFileNCout, " for: ", variableName))
# data <- ncvar_def( variableName, variableInfo[[variableName]]$units, list(dimX,dimY,dimT), FillValue, prec="float", chunksizes = c(length(RData$xyCoords$x),length(RData$xyCoords$y),1) compression=4)
data <- ncvar_def( variableName, variableInfo[[variableName]]$units, list(dimX,dimY,dimT), FillValue, prec="float")#, compression=4)
ncid_out <- ncvar_add( ncid_out, data )
}
ncvar_put( ncid_out, data, NCdata )
ncatt_put( ncid_out, variableName, "standard_name", variableInfo[[variableName]]$standardName)
ncatt_put( ncid_out, variableName, "long_name", variableInfo[[variableName]]$longName)
ncatt_put( ncid_out, variableName, "_FillValue", FillValue)
firstVar<-FALSE
}
nc_close(ncid_out)
} ## end initMonth
} ## end initYear
}
SCRIPTS/doReorder_combYears_combMembers_sepLeadmonths.R
View file @ b7a9c42b
......@@ -11,26 +11,26 @@ source(file = "./functions/infoGeneral.R")
submitscript <- FALSE
if (submitscript) {
members <- c(1:15)
initYears <-c(1981:2010)
initMonths <- c(X:X)
locName <- 'X'
resolution<-"0.75"
inPath<-sprintf("../DATA/System4_seasonal_15_rev1.0/%s_%s", locName, resolution)
outPath<-sprintf("../DATA/System4_seasonal_15_rev1.1REF2/%sdeg/%s_noBC", resolution, locName)
members <- c(1:15)
initYears <- c(1981:2010)
initMonths <- c(X:X)
locName <- 'X'
resolution <- 'X'
inPath <- sprintf("../DATA/System4_seasonal_15_rev1.0/%s_%s", locName, resolution)
outPath <- sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_noBC", resolution, locName)
} else {
members <-c(1:2)
initYears <-c(1981:1982)
initMonths<-1
#locName<-"GHA"
locName<-"EU"
resolution<-"0.75"
inPath<-sprintf("../DATA/System4_seasonal_15_rev1.0/%s_%s", locName, resolution)
outPath<-sprintf("../DATA/System4_seasonal_15_rev1.x/%sdeg/%s_noBC", resolution, locName)
members <- c(1:2)
initYears <- c(1981:1982)
initMonths <- 1
#locName <- "GHA"
locName <- "EU"
resolution <- "0.75"
inPath <- sprintf("../DATA/System4_seasonal_15_rev1.0/%s_%s", locName, resolution)
outPath <- sprintf("../DATA/System4_seasonal_15_rev1.x/%sdeg/%s_noBC", resolution, locName)
}
variables<-names(variableInfo)
#variables<-c( "pr" , "psl" , "rsds" , "rlds" , "tasmin","tasmax", "huss" , "sfcWind")
dir.create(outPath, recursive = TRUE, showWarnings = FALSE)
leadMonths<-c(0:6)
......@@ -81,8 +81,8 @@ for (initMonth in initMonths) {
## add some extra attributes
attr(RData,"contact") <- "Wietse Franssen (wietse.franssen@wur.nl)"
save(RData, file = paste0(oPrefix, ".RData"))
#R2Netcdf(paste0(oPrefix, ".nc4"), RData)
#save(RData, file = paste0(oPrefix, ".RData"))
R2Netcdf(paste0(oPrefix, ".nc4"), RData)
}
}
}
SCRIPTS/doReorder_sepYears_sepMembers_combLeadmonths.R
View file @ b7a9c42b
......@@ -18,28 +18,30 @@ if (submitscript) {
initMonths <- c(X:X)
locName <- 'X'
resolution<-"0.50"
bcInfo<-"BC"
# bcInfo<-"noBC"
# bcInfo<-"BC"
bcInfo<-"noBC"
inPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_%s", resolution, locName, bcInfo)
outPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_%s_final3", resolution, locName, bcInfo)
outPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_%s_modelFormat_perVariable", resolution, locName, bcInfo)
} else {
members <-c(1:15)
initYears <-c(1981:2010)
initMonths<-c(1:1)
initMonths<-c(1:12)
#locName<-"GHA"
locName<-"EU"
resolution<-"0.50"
inPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_noBC", resolution, locName)
outPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_noBC_out", resolution, locName)
# bcInfo<-"BC"
bcInfo<-"noBC"
inPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_%s", resolution, locName, bcInfo)
outPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_%s_modelFormat_perVariable", resolution, locName, bcInfo)
}
mask<-Netcdf2R(inFile = sprintf("../DATA/mask/mask_wfdei_%s.nc4", locName),"mask")
variables<-names(variableInfo)
variables<-c( "pr" , "psl" , "rsds" , "rlds" , "tas","tasmin","tasmax", "huss" , "sfcWind")
variables<-c( "rsds", "rlds", "tas", "tasmin", "tasmax", "huss", "sfcWind")
#variables<-c( "tas")
variables<-c( "tas")
#variables<-c( "pr")
dir.create(outPath, recursive = TRUE, showWarnings = FALSE)
variables<-c( "rsds", "rlds")
leadMonths<-c(0:6)
......@@ -55,6 +57,8 @@ nMembers<-length(members)
print("start")
for (initMonth in initMonths) {
for (variableName in variables) {
outPathTmp<-sprintf("%s/%s/", outPath, variableName)
dir.create(outPathTmp, recursive = TRUE, showWarnings = FALSE)
allocatedOutputArray<-FALSE
for (leadMonth in leadMonths) {
......@@ -71,8 +75,8 @@ for (initMonth in initMonths) {
targetSYear, targetEYear,
targetMonth, leadMonth)
print(iPrefix)
#load(sprintf("%s.RData",iPrefix))
RData<-Netcdf2R(sprintf("%s.nc4",iPrefix),variableName)
load(sprintf("%s.RData",iPrefix))
#RData<-Netcdf2R(sprintf("%s.nc4",iPrefix),variableName)
## Apply mask
print("Apply mask")
......@@ -83,7 +87,11 @@ for (initMonth in initMonths) {
}
## Convert Units
RData<-convert(RData, toUnit = variableInfo[[variableName]]$units)
if (variableName == "rsds" || variableName == "rlds") {
RData<-convert(RData, fromUnit = "1 W m-2", toUnit = variableInfo[[variableName]]$units)
} else {
RData<-convert(RData, toUnit = variableInfo[[variableName]]$units)
}
if (!allocatedOutputArray) {
## allocate arrays
......@@ -130,7 +138,7 @@ for (initMonth in initMonths) {
RData$InitializationDates<-finalList[[iLM]]$InitializationDates[which(initYears==initYear)]
## Save to file...
outFile <- sprintf("%s/%s_forcing_seas15_%s_%s_E%02d_%04d_%02d", outPath, variableName, locName, bcInfo, iMember, initYear, initMonth)
outFile <- sprintf("%s/%s_forcing_seas15_%s_%s_E%02d_%04d_%02d", outPathTmp, variableName, locName, bcInfo, iMember, initYear, initMonth)
#print(outFile)
R2Netcdf(outFile = sprintf("%s.nc4",outFile), RData)
}
......
SCRIPTS/jobScriptBiascorrection
View file @ b7a9c42b
......@@ -3,7 +3,7 @@
#SBATCH --time=5000
#SBATCH --mem=16024
#SBATCH --ntasks=1
#SBATCH --output=./log/log_biascorrection_<location>_targetMonth<targetMonth>_%j.txt
#SBATCH --output=./log/log_biascorrection_<location>_<resolution>deg_targetMonth<targetMonth>_%j.txt
#SBATCH --job-name=bia
#SBATCH --partition=ESG_Std
#SBATCH --mail-type=FAIL
......@@ -17,6 +17,6 @@ module load R/3.2.0
#
# run targetent job
echo '******** R-data Download...'
Rscript ./tmpScripts/doBiascorrection_<location>_targetMonth<targetMonth>.R
Rscript ./tmpScripts/doBiascorrection_<location>_<resolution>deg_targetMonth<targetMonth>.R
date
SCRIPTS/jobScriptNetcdfCombine 0 → 100644
View file @ b7a9c42b
#!/bin/bash
#SBATCH --account=5120867-01
#SBATCH --time=5000
#SBATCH --mem=4024
#SBATCH --ntasks=1
#SBATCH --output=./log/log_doNetcdfCombine_<location>_initMonth<initMonth>_%j.txt
#SBATCH --job-name=Combine
#SBATCH --partition=ESG_Std
#SBATCH --mail-type=FAIL
#SBATCH --mail-user=wietse.franssen@wur.nl
# print date and time
date
#
# print the working directory
pwd
module load R/3.2.0
#
# run current job
echo '******** start Rscript...'
Rscript ./tmpScripts/doNetcdfCombine_<location>_initMonth<initMonth>.R
date
SCRIPTS/jobScriptReorder_combYears_combMembers_sepLeadmonths
View file @ b7a9c42b
#!/bin/bash
#SBATCH --account=5120867-01
#SBATCH --time=5000
#SBATCH --mem=32024
#SBATCH --mem=16024
#SBATCH --ntasks=1
#SBATCH --output=./log/log_doReorder_combYears_combMembers_sepLeadmonths_<location>_initMonth<initMonth>_%j.txt
#SBATCH --output=./log/log_doReorder_combYears_combMembers_sepLeadmonths_<location>_<resolution>deg_initMonth<initMonth>_%j.txt
#SBATCH --job-name=Reformat
#SBATCH --partition=ESG_Std
#SBATCH --mail-type=FAIL
......@@ -17,6 +17,6 @@ module load R/3.2.0
#
# run current job
echo '******** start Rscript...'
Rscript ./tmpScripts/doReorder_combYears_combMembers_sepLeadmonths_<location>_initMonth<initMonth>.R
Rscript ./tmpScripts/doReorder_combYears_combMembers_sepLeadmonths_<location>_<resolution>deg_initMonth<initMonth>.R
date
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment