Commit e549609d authored by Franssen, Wietse's avatar Franssen, Wietse
Browse files

improved step 4

parent 0f589a55
......@@ -5,7 +5,7 @@
#SBATCH --ntasks=1
#SBATCH --output=./log/log_2_doReorder_combYears_combMembers_sepLeadmonths_<location>_<bcInfo>_<resolution>deg_initMonth<initMonth>_%j.txt
#SBATCH --job-name=Reorder2bias
#SBATCH --partition=ESG_High
#SBATCH --partition=ESG_Std
#SBATCH --mail-type=FAIL
#SBATCH --mail-user=wietse.franssen@wur.nl
# print date and time
......
#!/bin/bash
#SBATCH --account=5120867-01
#SBATCH --time=5000
#SBATCH --mem=16024
#SBATCH --mem=32024
#SBATCH --ntasks=1
#SBATCH --output=./log/log_3_biascorrection_<location>_<resolution>deg_targetMonth<targetMonth>_%j.txt
#SBATCH --job-name=bia
......
rm(list=ls())
library(fields) # e.g: using the fields library
library(abind)
library(ncdf4)
library(ecomsUDG.Raccess)
source(file = "./functions/functionsGeneral.R")
source(file = "./functions/functionReformat.R")
source(file = "./functions/functionR2Netcdf.R")
source(file = "./functions/functionNetcdf2R.R")
source(file = "./functions/functionConvert.R")
source(file = "./functions/infoGeneral.R")
showMemoryUse <- function(sort="size", decreasing=FALSE, limit) {
objectList <- ls(parent.frame())
oneKB <- 1024
oneMB <- 1048576
oneGB <- 1073741824
memoryUse <- sapply(objectList, function(x) as.numeric(object.size(eval(parse(text=x)))))
memListing <- sapply(memoryUse, function(size) {
if (size >= oneGB) return(paste(round(size/oneGB,2), "GB"))
else if (size >= oneMB) return(paste(round(size/oneMB,2), "MB"))
else if (size >= oneKB) return(paste(round(size/oneKB,2), "kB"))
else return(paste(size, "bytes"))
})
memListing <- data.frame(objectName=names(memListing),memorySize=memListing,row.names=NULL)
if (sort=="alphabetical") memListing <- memListing[order(memListing$objectName,decreasing=decreasing),]
else memListing <- memListing[order(memoryUse,decreasing=decreasing),] #will run if sort not specified or "size"
if(!missing(limit)) memListing <- memListing[1:limit,]
print(memListing, row.names=FALSE)
return(invisible(memListing))
}
submitscript <- FALSE
if (submitscript) {
......@@ -45,37 +15,24 @@ if (submitscript) {
locName <- 'X'
resolution <- 'X'
bcInfo <- 'X'
inPath <- sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_%s", resolution, locName, bcInfo)
outPath <- sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_%s_modelFormat_perVariable", resolution, locName, bcInfo)
inPath <- sprintf("../DATA/System4_seasonal_15_rev1.2/%sdeg/%s_%s_biasformat", resolution, locName, bcInfo)
outPath <- sprintf("../DATA/System4_seasonal_15_rev1.2/%sdeg/%s_%s", resolution, locName, bcInfo)
} else {
members <- c(1:15)
initYears <- c(1981:2010)
initMonths <- c(1:12)
#locName <- "GHA"
locName <- "EU"
resolution <- "0.50"
# bcInfo <- "BC"
bcInfo <- "noBC"
inPath <- sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_%s", resolution, locName, bcInfo)
outPath <- sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_%s_modelFormat_perVariable", resolution, locName, bcInfo)
locName <- "GHA"
#locName <- "EU"
resolution <- "0.75"
bcInfo <- "BC"
inPath <- sprintf("../DATA/System4_seasonal_15_rev1.2/%sdeg/%s_%s_biasformat", resolution, locName, bcInfo)
outPath <- sprintf("../DATA/System4_seasonal_15_rev1.2/%sdeg/%s_%s", resolution, locName, bcInfo)
}
mask<-Netcdf2R(inFile = sprintf("../DATA/mask/mask_wfdei_%s.nc4", locName),"mask")
variables<-names(variableInfo)
variables<-c( "rsds", "rlds", "pr", "tas", "tasmin", "tasmax", "huss", "sfcWind")
variables<-c( "rsds", "rlds")
leadMonths<-c(0:6)
sYear<-initYears[1]
eYear<-initYears[length(initYears)]
nYears<-length(initYears)
sMember<-members[1]
eMember<-members[length(members)]
nMembers<-length(members)
print("start")
for (initMonth in initMonths) {
for (variableName in variables) {
......@@ -84,13 +41,12 @@ for (initMonth in initMonths) {
allocatedOutputArray<-FALSE
for (leadMonth in leadMonths) {
indexesYearsLeadMonths<-indexesForYearsPerLeadMonth(sYear,eYear,initMonth)
indexesYearsLeadMonths<-indexesForYearsPerLeadMonth(initYears[1],initYears[length(initYears)],initMonth)
targetMonth <- getInitTargetInfo( initYear = 0000, initMonth = initMonth, leadMonth = leadMonth )$targetMonth
targetSYear <- getInitTargetInfo( initYear = initYears[1], initMonth = initMonth, leadMonth = leadMonth )$targetYear
targetEYear <- getInitTargetInfo( initYear = initYears[length(initYears)], initMonth = initMonth, leadMonth = leadMonth )$targetYear
# print(sprintf("targetmonth: %s, leadMonth: %s",targetMonth, leadMonth))
iPrefix <- sprintf("%s/%s_forcing_seas15_%s_%s_E%02d-%02d_TAR%4d-%4d_%02d_LM%d",
inPath, variableName, locName, bcInfo,
members[1], members[length(members)],
......@@ -98,26 +54,10 @@ for (initMonth in initMonths) {
targetMonth, leadMonth)
print(iPrefix)
#load(sprintf("%s.RData",iPrefix))
RData<-Netcdf2R(sprintf("%s.nc4",iPrefix),variableName)
# gc(TRUE)
## Apply mask
print("Apply mask")
for (iMember in 1:members) {
for (iTime in 1:dim(RData$Data)[2]) {
RData$Data[iMember,iTime,,]<-RData$Data[iMember,iTime,,]*mask$Data[1,,]
}
}
# gc(reset = TRUE)
# showMemoryUse(decreasing=TRUE, limit=5)
## Convert Units
if (variableName == "rsds" || variableName == "rlds") {
RData<-convert(RData, fromUnit = "1 W m-2", toUnit = variableInfo[[variableName]]$units)
} else {
RData<-convert(RData, toUnit = variableInfo[[variableName]]$units)
}
RData<-Netcdf2R(sprintf("%s.nc4",iPrefix),variableName)
# gc(reset = TRUE)
# showMemoryUse(decreasing=TRUE, limit=5)
if (!allocatedOutputArray) {
## allocate arrays
finalList<-NULL
......@@ -125,9 +65,9 @@ for (initMonth in initMonths) {
for (iLM in 1:7) {
finalList[[iLM]] <- RData
listNames<-c(listNames,paste0("LeadMonth_",iLM-1))
totalDays<-indexesYearsLeadMonths$eIndexes[as.character(eYear),iLM]
totalDays<-indexesYearsLeadMonths$eIndexes[as.character(initYears[length(initYears)]),iLM]
finalList[[iLM]]$Data<-array(NA,dim=c(nMembers,totalDays,length(RData$xyCoords$y),length(RData$xyCoords$x)))
finalList[[iLM]]$Data<-array(NA,dim=c(length(members),totalDays,length(RData$xyCoords$y),length(RData$xyCoords$x)))
finalList[[iLM]]$Dates$start<-character(length=totalDays)
finalList[[iLM]]$Dates$end<-character(length=totalDays)
}
......@@ -152,7 +92,6 @@ for (initMonth in initMonths) {
for (iMember in members) {
## Fill with data
for (iLM in 1:7) {
#print(iLM)
month<-indexes[iLM,"month"]
year<-indexes[iLM,"year"]
datesInCurrMonth <- seq(as.Date(as.Date(sprintf("%04d-%02d-01", year, month))),by = "1 day", length=indexes[iLM,"nDaysTotalInMonth"])
......@@ -166,7 +105,6 @@ for (initMonth in initMonths) {
## Save to file...
outFile <- sprintf("%s/%s_forcing_seas15_%s_%s_E%02d_%04d_%02d", outPathTmp, variableName, locName, bcInfo, iMember, initYear, initMonth)
#print(outFile)
R2Netcdf(outFile = sprintf("%s.nc4",outFile), RData)
}
}
......
......@@ -3,9 +3,9 @@
#SBATCH --time=5000
#SBATCH --mem=16024
#SBATCH --ntasks=1
#SBATCH --output=./log/log_doReorder_sepYears_sepMembers_combLeadmonths_<location>_<bcInfo>_<resolution>deg_initMonth<initMonth>_%j.txt
#SBATCH --output=./log/log_4_doReorder_sepYears_sepMembers_combLeadmonths_<location>_<bcInfo>_<resolution>deg_initMonth<initMonth>_%j.txt
#SBATCH --job-name=Reorder
#SBATCH --partition=ESG_High
#SBATCH --partition=ESG_Std
#SBATCH --mail-type=FAIL
#SBATCH --mail-user=wietse.franssen@wur.nl
# print date and time
......@@ -17,10 +17,7 @@ module load R/3.2.0
#
# run current job
echo '******** start Rscript...'
#ulimit -a
#ulimit -v 16000000
#ulimit -a
Rscript ./tmpScripts/doReorder_sepYears_sepMembers_combLeadmonths_<location>_<bcInfo>_<resolution>deg_initMonth<initMonth>.R
Rscript ./tmpScripts/4_doReorder_sepYears_sepMembers_combLeadmonths_<location>_<bcInfo>_<resolution>deg_initMonth<initMonth>.R
date
#!/bin/bash
tmpScripts="./tmpScripts"
#location="EU"; resolution="0.50"
location="GHA"; resolution="0.50"
#location="GHA"; resolution="0.75"
#location="GHA"; resolution="0.50"
location="GHA"; resolution="0.75"
bcInfo="noBC"
#bcInfo="BC"
#bcInfo="noBC"
bcInfo="BC"
mkdir -p $tmpScripts
for iInitMonth in $(seq -f "%02g" 1 1); do
for iInitMonth in $(seq -f "%02g" 1 12); do
echo "leadMonth: $iInitMonth"
sed -e "s|<initMonth>|$iInitMonth|g" \
-e "s|<location>|$location|g" \
-e "s|<resolution>|$resolution|g" \
-e "s|<bcInfo>|$bcInfo|g" \
jobScriptReorder_sepYears_sepMembers_combLeadmonths > $tmpScripts"/jobScriptReorder_sepYears_sepMembers_combLeadmonths_"$location"_"$bcInfo"_"$resolution"deg_initMonth"$iInitMonth
4_jobScriptReorder_sepYears_sepMembers_combLeadmonths > $tmpScripts"/4_jobScriptReorder_sepYears_sepMembers_combLeadmonths_"$location"_"$bcInfo"_"$resolution"deg_initMonth"$iInitMonth
sed -e "s|submitscript <- FALSE|submitscript <- TRUE|g" \
-e "s|initMonths <- c(X:X)|initMonths <- c($iInitMonth:$iInitMonth)|g" \
-e "s|locName <- 'X'|locName <- '$location'|g" \
-e "s|resolution <- 'X'|resolution <- '$resolution'|g" \
-e "s|bcInfo <- 'X'|bcInfo <- '$bcInfo'|g" \
doReorder_sepYears_sepMembers_combLeadmonths.R > $tmpScripts"/doReorder_sepYears_sepMembers_combLeadmonths_"$location"_"$bcInfo"_"$resolution"deg_initMonth"$iInitMonth".R"
sbatch "$tmpScripts/jobScriptReorder_sepYears_sepMembers_combLeadmonths_"$location"_"$bcInfo"_"$resolution"deg_initMonth"$iInitMonth
4_doReorder_sepYears_sepMembers_combLeadmonths.R > $tmpScripts"/4_doReorder_sepYears_sepMembers_combLeadmonths_"$location"_"$bcInfo"_"$resolution"deg_initMonth"$iInitMonth".R"
sbatch "$tmpScripts/4_jobScriptReorder_sepYears_sepMembers_combLeadmonths_"$location"_"$bcInfo"_"$resolution"deg_initMonth"$iInitMonth
done
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment