fixed step 1b and step 2

SCRIPTS/1b_doReformatOldData.R

+rm(list=ls())
+source(file = "./functions/functionsGeneral.R")
+source(file = "./functions/functionR2Netcdf.R")
+source(file = "./functions/functionNetcdf2R.R")
+source(file = "./functions/functionConvert.R")
+source(file = "./functions/infoGeneral.R")
+
+submitscript <- FALSE
+
+if (submitscript) {
+  members      <- c(1:15)
+  initYears    <- c(1981:2010)
+  initMonths   <- c(X:X)
+  locName      <- 'X'
+  resolution   <- 'X'
+  inPath       <- sprintf("../DATA/System4_seasonal_15_rev1.0/%s_%s", locName, resolution)
+  outPath      <- sprintf("../DATA/System4_seasonal_15_rev1.2/%sdeg/%s_noBC", resolution, locName)
+} else {
+  members      <- c(1:2)
+  initYears    <- c(1981:1982)
+  initMonths   <- 1
+  locName      <- "GHA"
+  #locName      <- "EU"
+  resolution   <- "0.75"
+  inPath       <- sprintf("../DATA/System4_seasonal_15_rev1.0/%s_%s", locName, resolution)
+  outPath      <- sprintf("../DATA/System4_seasonal_15_rev1.2/%sdeg/%s_noBC", resolution, locName)
+}
+
+mask<-Netcdf2R(inFile = sprintf("../DATA/mask/mask_wfdei_%s_%s.nc4", locName, resolution), "mask")
+
+variables<-names(variableInfo)
+
+print("start")
+for (variableName in variables) {
+  dir.create(sprintf("%s/%s", outPath, variableName), recursive = TRUE, showWarnings = FALSE)
+  for (initMonth in initMonths) {
+    for (iMember in members) {
+      for (initYear in initYears) {
+        if ('ecomsName' %in% names(variableInfo[[variableName]])) {
+          variableNameECOMS<-variableInfo[[variableName]]$ecomsName
+        } else {
+          variableNameECOMS<-variableName
+        }
+        
+        ## Load file...
+        inFile <- sprintf("%s/%s/%s_forcing_seas15_%s_noBC_E%02d_%04d_%02d.RData", inPath, variableNameECOMS, variableNameECOMS, locName, iMember, initYear, initMonth)
+        print(sprintf("Reading: %s",inFile))
+        load(file = inFile)
+        RData<-sys4
+        rm(sys4)
+        
+        # Check units
+        RData$Variable$varName<-variableName
+        attr(RData$Variable,"standard_name")<-variableInfo[[variableName]]$standardName
+        attr(RData$Variable,"long_name")<-variableInfo[[variableName]]$longName
+        attr(RData$Variable,"units")<-variableInfo[[variableName]]$unitsEcoms
+        
+        RData$xyCoords$x[]<-round(RData$xyCoords$x[],2)
+        RData$xyCoords$y[]<-round(RData$xyCoords$y[],2)
+        
+        ## set all negative precipitation values 0
+        if (variableName == "pr") {
+          RData$Data[RData$Data < 0] <- 0
+        }
+        
+      ## Convert Units
+      if (variableName == "rsds" || variableName == "rlds") {
+        RData<-convert(RData, fromUnit = "1 W m-2", toUnit = variableInfo[[variableName]]$units)
+      } else {
+        RData<-convert(RData, toUnit = variableInfo[[variableName]]$units)
+        # RData<-convert(RData, fromUnit = variableInfo[[variableName]]$unitsEcoms, toUnit = variableInfo[[variableName]]$units)
+      }
+
+        ## check if grids are the same as mask
+        ## if not: make the same as mask
+        if (RData$xyCoords$x[1] != locationInfo[[sprintf("res%s",resolution)]][[locName]]$lonmin ||
+            RData$xyCoords$x[length(RData$xyCoords$x)] != locationInfo[[sprintf("res%s",resolution)]][[locName]]$lonmax ||
+            RData$xyCoords$y[1] != locationInfo[[sprintf("res%s",resolution)]][[locName]]$latmin ||
+            RData$xyCoords$y[length(RData$xyCoords$y)] != locationInfo[[sprintf("res%s",resolution)]][[locName]]$latmax) {
+          
+          indexesLat<-c(which(RData$xyCoords$y==mask$xyCoords$y[1]):which(RData$xyCoords$y==mask$xyCoords$y[length(mask$xyCoords$y)]))
+          indexesLon<-c(which(RData$xyCoords$x==mask$xyCoords$x[1]):which(RData$xyCoords$x==mask$xyCoords$x[length(mask$xyCoords$x)]))
+          RData$xyCoords$x<-RData$xyCoords$x[indexesLon]
+          RData$xyCoords$y<-RData$xyCoords$y[indexesLat]
+          attr(RData$Data,"projection") <- "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs +towgs84=0,0,0"
+          RData$Data<-RData$Data[,indexesLat,indexesLon]
+          attr(RData$Data,"dimensions") <- c("time","lat","lon")
+        }
+        
+        ## Apply mask
+        for (iTime in 1:dim(RData$Data)[2]) {
+          RData$Data[iTime,,]<-RData$Data[iTime,,]*mask$Data[1,,]
+        }
+        
+        ## add some extra attributes
+        attr(RData,"contact") <- "Wietse Franssen (wietse.franssen@wur.nl)"
+        
+        ## Save file...
+        outPrefix <- sprintf("%s/%s/%s_forcing_seas15_%s_noBC_E%02d_INIT%04d_%02d", outPath, variableName, variableName, locName, iMember, initYear, initMonth)
+        # print(outPrefix)
+        R2Netcdf(paste0(outPrefix, ".nc4"), RData)
+      }
+    }
+  }
+}

SCRIPTS/1b_jobScriptReformatOldData

+#!/bin/bash
+#SBATCH --account=5120867-01
+#SBATCH --time=5000
+#SBATCH --mem=4024
+#SBATCH --ntasks=1
+#SBATCH --output=./log/log_1b_doReformatOldData_<location>_<resolution>deg_initMonth<initMonth>_%j.txt
+#SBATCH --job-name=Reformat
+#SBATCH --partition=ESG_Std
+#SBATCH --mail-type=FAIL
+#SBATCH --mail-user=wietse.franssen@wur.nl
+# print date and time
+date                                                                                                   
+#
+# print the working directory
+pwd
+module load R/3.2.0
+#
+# run current job
+echo '******** start Rscript...'
+Rscript ./tmpScripts/1b_doReformatOldData_<location>_<resolution>deg_initMonth<initMonth>.R
+date
+

SCRIPTS/1b_submit_reformatOldData.sh

+#!/bin/bash
+tmpScripts="./tmpScripts"
+#location="EU";  resolution="0.50"
+location="GHA"; resolution="0.50"
+#location="GHA"; resolution="0.75"
+
+mkdir -p $tmpScripts
+
+for iInitMonth in $(seq -f "%02g" 1 12); do
+  echo "initMonth: $iInitMonth"
+  sed -e "s|<initMonth>|$iInitMonth|g" \
+      -e "s|<location>|$location|g" \
+      -e "s|<resolution>|$resolution|g" \
+      1b_jobScriptReformatOldData > $tmpScripts"/1b_jobScriptReformatOldData_"$location"_"$resolution"deg_initMonth"$iInitMonth
+  sed -e "s|submitscript <- FALSE|submitscript <- TRUE|g" \
+      -e "s|initMonths   <- c(X:X)|initMonths   <- c($iInitMonth:$iInitMonth)|g" \
+      -e "s|locName      <- 'X'|locName      <- '$location'|g" \
+      -e "s|resolution   <- 'X'|resolution   <- '$resolution'|g" \
+      1b_doReformatOldData.R > $tmpScripts"/1b_doReformatOldData_"$location"_"$resolution"deg_initMonth"$iInitMonth".R"
+  sbatch "$tmpScripts/1b_jobScriptReformatOldData_"$location"_"$resolution"deg_initMonth"$iInitMonth
+done

SCRIPTS/2_submit_regrid.sh

-#!/bin/bash
-tmpScripts="./tmpScripts"
-#location="GHA"
-location="EU"
-mkdir -p $tmpScripts
-
-for iTargetMonth in $(seq -f "%02g" 1 12); do
-  echo "leadMonth: $iTargetMonth"
-  sed -e "s|<targetMonth>|$iTargetMonth|g" \
-      -e "s|<location>|$location|g" \
-      jobScriptRegrid > $tmpScripts"/jobScriptRegrid_"$location"_targetMonth"$iTargetMonth
-  sed -e "s|submitscript <- FALSE|submitscript <- TRUE|g" \
-      -e "s|targetMonths <- c(X:X)|targetMonths <- c($iTargetMonth:$iTargetMonth)|g" \
-      -e "s|locName    <- 'X'|locName    <- '$location'|g" \
-      doRegrid.R > $tmpScripts"/doRegrid_"$location"_targetMonth"$iTargetMonth".R"
-  sbatch "$tmpScripts/jobScriptRegrid_"$location"_targetMonth"$iTargetMonth
-done

SCRIPTS/4_submit_biascorrection.sh → SCRIPTS/3_submit_biascorrection.sh

 #!/bin/bash
 tmpScripts="./tmpScripts"
 #location="EU";  resolution="0.50"
-#location="GHA"; resolution="0.50"
-location="GHA"; resolution="0.75"
+location="GHA"; resolution="0.50"
+#location="GHA"; resolution="0.75"
 
 mkdir -p $tmpScripts
 
@@ -14,8 +14,8 @@ for iTargetMonth in $(seq -f "%02g" 1 12); do
       jobScriptBiascorrection > $tmpScripts"/jobScriptBiascorrection_"$location"_"$resolution"deg_targetMonth"$iTargetMonth
   sed -e "s|submitscript <- FALSE|submitscript <- TRUE|g" \
       -e "s|targetMonths <- c(X:X)|targetMonths  <- c($iTargetMonth:$iTargetMonth)|g" \
-      -e "s|locName     <- 'X'|locName     <- '$location'|g" \
-      -e "s|resolution  <- 'X'|resolution  <- '$resolution'|g" \
+      -e "s|locName      <- 'X'|locName      <- '$location'|g" \
+      -e "s|resolution   <- 'X'|resolution   <- '$resolution'|g" \
       doBiascorrection.R > $tmpScripts"/doBiascorrection_"$location"_"$resolution"deg_targetMonth"$iTargetMonth".R"
   sbatch "$tmpScripts/jobScriptBiascorrection_"$location"_"$resolution"deg_targetMonth"$iTargetMonth
 done

SCRIPTS/5_submit_reorder_sepYears_sepMembers_combLeadmonths.sh

 #!/bin/bash
 tmpScripts="./tmpScripts"
-#location="GHA"
-location="EU"
+#location="EU";  resolution="0.50"
+location="GHA"; resolution="0.50"
+#location="GHA"; resolution="0.75"
+
+bcInfo="noBC"
+#bcInfo="BC"
+
 mkdir -p $tmpScripts
 
-for iInitMonth in $(seq -f "%02g" 1 12); do
+for iInitMonth in $(seq -f "%02g" 1 1); do
   echo "leadMonth: $iInitMonth"
   sed -e "s|<initMonth>|$iInitMonth|g" \
       -e "s|<location>|$location|g" \
-      jobScriptReorder_sepYears_sepMembers_combLeadmonths > $tmpScripts"/jobScriptReorder_sepYears_sepMembers_combLeadmonths_"$location"_initMonth"$iInitMonth
+      -e "s|<resolution>|$resolution|g" \
+      -e "s|<bcInfo>|$bcInfo|g" \
+      jobScriptReorder_sepYears_sepMembers_combLeadmonths > $tmpScripts"/jobScriptReorder_sepYears_sepMembers_combLeadmonths_"$location"_"$bcInfo"_"$resolution"deg_initMonth"$iInitMonth
   sed -e "s|submitscript <- FALSE|submitscript <- TRUE|g" \
-      -e "s|initMonths  <- c(X:X)|initMonths  <- c($iInitMonth:$iInitMonth)|g" \
-      -e "s|locName    <- 'X'|locName    <- '$location'|g" \
-      doReorder_sepYears_sepMembers_combLeadmonths.R > $tmpScripts"/doReorder_sepYears_sepMembers_combLeadmonths_"$location"_initMonth"$iInitMonth".R"
-  sbatch "$tmpScripts/jobScriptReorder_sepYears_sepMembers_combLeadmonths_"$location"_initMonth"$iInitMonth
+      -e "s|initMonths   <- c(X:X)|initMonths   <- c($iInitMonth:$iInitMonth)|g" \
+      -e "s|locName      <- 'X'|locName      <- '$location'|g" \
+      -e "s|resolution   <- 'X'|resolution   <- '$resolution'|g" \
+      -e "s|bcInfo       <- 'X'|bcInfo       <- '$bcInfo'|g" \
+      doReorder_sepYears_sepMembers_combLeadmonths.R > $tmpScripts"/doReorder_sepYears_sepMembers_combLeadmonths_"$location"_"$bcInfo"_"$resolution"deg_initMonth"$iInitMonth".R"
+  sbatch "$tmpScripts/jobScriptReorder_sepYears_sepMembers_combLeadmonths_"$location"_"$bcInfo"_"$resolution"deg_initMonth"$iInitMonth
 done

SCRIPTS/README.txt

-## Step 1 (If using old data)
-reformat the old 0.75 degree data (seperate years, seperate members, combined leadmonths) to new data (combined years, combined members, seperate leadmonths)
-
-## Step 1 (If downloading new data)
+## Step 1a (If downloading new data)
 download the 0.75 degree data
-NOTE1:  downloading combined years, combined members, seperate leadmonths gives strange timings in with the variables "tp", "rsds" and "rlds"
+download and regrid the 0.50 degree data
+NOTE1: downloading combined years, combined members, seperate leadmonths gives strange timings in with the variables "tp", "rsds" and "rlds"
        it is better to download leadmonth0 and the season 1till7 in one go.
-NOTE2a: even if you download the correct way (see NOTE1), you should remove timestep 1 of leadmonth0 from "rsds" and "rlds" 
-NOTE2b: even if you download the correct way (see NOTE1), you should move all data one timestep backwards for "pr" (because t1 are all zero)
+NOTE2: even if you download the correct way (see NOTE1), you should remove timestep 1 of leadmonth0 from "rsds" and "rlds" 
+NOTE3: even if you download the correct way (see NOTE1), you should move all data one timestep backwards for "pr" (because t1 are all zero)
+
+## Step 1b (If using old data)
+reorder the old R-data to the Netcdf format.
+the reordering script:
+* selects the right region if needed (based on the mask file)
+* applies a landsea mask (based on the mask file)
+* removes values lower than zero (only for precipitation)
+* converts to ALMA (model) units.
 
 ## Step 2
-regrid the 0.75 degree data into 0.50 degree data
-both datasets are: combined years, combined members, seperate leadmonths
+converts from:
+* seperate years, seperate members, combined leadmonths 
+to:
+* combined years, combined members, seperate leadmonths
 
 ## Step 3
-biascorrect both 0.75 and 0.50 degree data
+biascorrection
 all datasets are: combined years, combined members, seperate leadmonths

SCRIPTS/doBiascorrection.R

@@ -8,6 +8,7 @@ source(file = "./functions/infoGeneral.R")
 source(file = "./functions/functionMakeSelection.R")
 source(file = "./functions/functionBiascorrectionWUR.R")
 source(file = "./functions/functionR2Netcdf.R")
+source(file = "./functions/functionNetcdf2R.R")
 
 submitscript <- FALSE
 
@@ -17,7 +18,7 @@ if (submitscript) {
   targetMonths <- c(X:X)
   leadMonths   <- c(0:6)
   locName      <- 'X'
-  resolution   <- "0.50"
+  resolution   <- 'X'
   obsPath      <- sprintf("../DATA/wfdei_rev3.0/%sdeg/%s/", resolution, locName)
   inPath       <- sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_noBC", resolution, locName)
   outPath      <- sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_BC", resolution, locName)
@@ -47,16 +48,18 @@ for (variableName in variables) {
       targetSYear <- getInitTargetInfo( initYear = initYears[1], targetMonth = targetMonth, leadMonth = leadMonth )$targetYear
       targetEYear <- getInitTargetInfo( initYear = initYears[length(initYears)], targetMonth = targetMonth, leadMonth = leadMonth )$targetYear
       
-      iFile <- sprintf("%s/%s_forcing_seas15_%s_noBC_E%02d-%02d_TAR%4d-%4d_%02d_LM%d.RData",
+      iPrefix <- sprintf("%s/%s_forcing_seas15_%s_noBC_E%02d-%02d_TAR%4d-%4d_%02d_LM%d",
                        inPath, variableName, locName, 
                        members[1], members[length(members)],
                        targetSYear, targetEYear,
                        targetMonth, leadMonth)
 
-      print(sprintf("Open: %s", iFile))
-      load(iFile)
-      RData_noBC<-RData
-      rm(RData)
+      print(sprintf("Open: %s", iPrefix))
+      RData_noBC <- Netcdf2R(paste0(iPrefix, ".nc4"), variableName)
+
+      #load(file = paste0(iPrefix, ".RData"))
+      #RData_noBC<-RData
+      #rm(RData)
       
       if (variableName == "pr") {
         obsFile <- sprintf("%s/%s_wfdei_GPCC_%s_%4d-%4d_%02d.RData",

SCRIPTS/doReorder_combYears_combMembers_sepLeadmonths.R

@@ -28,6 +28,7 @@ if (submitscript) {
   inPath       <- sprintf("../DATA/System4_seasonal_15_rev1.0/%s_%s", locName, resolution)
   outPath      <- sprintf("../DATA/System4_seasonal_15_rev1.x/%sdeg/%s_noBC", resolution, locName)
 }
+mask<-Netcdf2R(inFile = sprintf("../DATA/mask/mask_wfdei_%s.nc4", locName),"mask")
 
 variables<-names(variableInfo)
 
@@ -72,6 +73,32 @@ for (initMonth in initMonths) {
       RData$xyCoords$x[]<-round(RData$xyCoords$x[],2)
       RData$xyCoords$y[]<-round(RData$xyCoords$y[],2)
       
+#      ## check if grids are the same
+#      if (RData$xyCoords$x[1] != locationInfo$res0.50[[locName]]$lonmin ||
+#          RData$xyCoords$x[length(RData$xyCoords$x)] != locationInfo$res0.50[[locName]]$lonmax ||
+#          RData$xyCoords$y[1] != locationInfo$res0.50[[locName]]$latmin ||
+#          RData$xyCoords$y[length(RData$xyCoords$y)] != locationInfo$res0.50[[locName]]$latmax) {
+#
+#        ## make grid
+#        newGrid<-getGrid(RData)
+#        newGrid$x <- c(locationInfo$res0.50[[locName]]$lonmin, locationInfo$res0.50[[locName]]$lonmax)
+#        newGrid$y <- c(locationInfo$res0.50[[locName]]$latmin, locationInfo$res0.50[[locName]]$latmax)
+#        attr(newGrid,"resX") <- 0.50
+#        attr(newGrid,"resY") <- 0.50
+#        ## regrid
+#        print(sprintf("Regridding..."))
+#        RData <- interpGridData(RData, new.grid = newGrid, method = "bilinear");
+#      }
+
+      ## Apply mask
+      print("Apply mask")
+      for (iMember in 1:members) {
+        for (iTime in 1:dim(RData$Data)[2]) {
+          RData$Data[iMember,iTime,,]<-RData$Data[iMember,iTime,,]*mask$Data[1,,]
+        }
+      }      
+
+      
       # Check units
       RData$Variable$varName<-variableName
       attr(RData$Variable,"standard_name")<-variableInfo[[variableName]]$standardName

SCRIPTS/doReorder_sepYears_sepMembers_combLeadmonths.R

@@ -9,38 +9,60 @@ source(file = "./functions/functionR2Netcdf.R")
 source(file = "./functions/functionNetcdf2R.R")
 source(file = "./functions/functionConvert.R")
 source(file = "./functions/infoGeneral.R")
+showMemoryUse <- function(sort="size", decreasing=FALSE, limit) {
 
+  objectList <- ls(parent.frame())
+
+  oneKB <- 1024
+  oneMB <- 1048576
+  oneGB <- 1073741824
+
+  memoryUse <- sapply(objectList, function(x) as.numeric(object.size(eval(parse(text=x)))))
+
+  memListing <- sapply(memoryUse, function(size) {
+        if (size >= oneGB) return(paste(round(size/oneGB,2), "GB"))
+        else if (size >= oneMB) return(paste(round(size/oneMB,2), "MB"))
+        else if (size >= oneKB) return(paste(round(size/oneKB,2), "kB"))
+        else return(paste(size, "bytes"))
+      })
+
+  memListing <- data.frame(objectName=names(memListing),memorySize=memListing,row.names=NULL)
+
+  if (sort=="alphabetical") memListing <- memListing[order(memListing$objectName,decreasing=decreasing),] 
+  else memListing <- memListing[order(memoryUse,decreasing=decreasing),] #will run if sort not specified or "size"
+
+  if(!missing(limit)) memListing <- memListing[1:limit,]
+
+  print(memListing, row.names=FALSE)
+  return(invisible(memListing))
+}
 submitscript <- FALSE
 
 if (submitscript) {
-  members    <- c(1:15)
-  initYears <-c(1981:2010)
-  initMonths  <- c(X:X)
-  locName    <- 'X'
-  resolution<-"0.50"
-#  bcInfo<-"BC"
-  bcInfo<-"noBC"
-  inPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_%s", resolution, locName, bcInfo)
-  outPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_%s_modelFormat_perVariable", resolution, locName, bcInfo)
+  members      <- c(1:15)
+  initYears    <- c(1981:2010)
+  initMonths   <- c(X:X)
+  locName      <- 'X'
+  resolution   <- 'X'
+  bcInfo       <- 'X'
+  inPath       <- sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_%s", resolution, locName, bcInfo)
+  outPath      <- sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_%s_modelFormat_perVariable", resolution, locName, bcInfo)
 } else {
-  members    <-c(1:15)
-  initYears <-c(1981:2010)
-  initMonths<-c(1:12)
-  #locName<-"GHA"
-  locName<-"EU"
-  resolution<-"0.50"
-#  bcInfo<-"BC"
-  bcInfo<-"noBC"
-  inPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_%s", resolution, locName, bcInfo)
-  outPath<-sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_%s_modelFormat_perVariable", resolution, locName, bcInfo)
+  members      <- c(1:15)
+  initYears    <- c(1981:2010)
+  initMonths   <- c(1:12)
+  #locName      <- "GHA"
+  locName      <- "EU"
+  resolution   <- "0.50"
+#  bcInfo       <- "BC"
+  bcInfo       <- "noBC"
+  inPath       <- sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_%s", resolution, locName, bcInfo)
+  outPath      <- sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_%s_modelFormat_perVariable", resolution, locName, bcInfo)
 }
 mask<-Netcdf2R(inFile = sprintf("../DATA/mask/mask_wfdei_%s.nc4", locName),"mask")
 
 variables<-names(variableInfo)
-variables<-c( "pr"    ,  "psl"    , "rsds"   , "rlds" , "tas","tasmin","tasmax", "huss" ,   "sfcWind")
-variables<-c( "rsds", "rlds", "tas", "tasmin", "tasmax", "huss", "sfcWind")
-variables<-c( "tas")
-#variables<-c( "pr")
+variables<-c( "rsds", "rlds", "pr", "tas", "tasmin", "tasmax", "huss", "sfcWind")
 variables<-c( "rsds", "rlds")
 
 leadMonths<-c(0:6)
@@ -75,8 +97,9 @@ for (initMonth in initMonths) {
                          targetSYear, targetEYear,
                          targetMonth, leadMonth)
       print(iPrefix)
-      load(sprintf("%s.RData",iPrefix))
-      #RData<-Netcdf2R(sprintf("%s.nc4",iPrefix),variableName)
+      #load(sprintf("%s.RData",iPrefix))
+      RData<-Netcdf2R(sprintf("%s.nc4",iPrefix),variableName)
+#      gc(TRUE)
       
       ## Apply mask
       print("Apply mask")
@@ -85,14 +108,16 @@ for (initMonth in initMonths) {
           RData$Data[iMember,iTime,,]<-RData$Data[iMember,iTime,,]*mask$Data[1,,]
         }
       }      
-      
+#      gc(reset = TRUE)
+#      showMemoryUse(decreasing=TRUE, limit=5)
       ## Convert Units
       if (variableName == "rsds" || variableName == "rlds") {
         RData<-convert(RData, fromUnit = "1 W m-2", toUnit = variableInfo[[variableName]]$units)
       } else {
         RData<-convert(RData, toUnit = variableInfo[[variableName]]$units)
       }
-      
+#      gc(reset = TRUE)
+#      showMemoryUse(decreasing=TRUE, limit=5)
       if (!allocatedOutputArray) {
         ## allocate arrays
         finalList<-NULL
@@ -102,7 +127,7 @@ for (initMonth in initMonths) {
           listNames<-c(listNames,paste0("LeadMonth_",iLM-1))
           totalDays<-indexesYearsLeadMonths$eIndexes[as.character(eYear),iLM]
           
-          finalList[[iLM]]$Data<-array(0,dim=c(nMembers,totalDays,length(RData$xyCoords$y),length(RData$xyCoords$x)))
+          finalList[[iLM]]$Data<-array(NA,dim=c(nMembers,totalDays,length(RData$xyCoords$y),length(RData$xyCoords$x)))
           finalList[[iLM]]$Dates$start<-character(length=totalDays)
           finalList[[iLM]]$Dates$end<-character(length=totalDays)
         }
@@ -110,6 +135,8 @@ for (initMonth in initMonths) {
         allocatedOutputArray<-TRUE
       }
       finalList[[paste0("LeadMonth_",leadMonth)]]<-RData
+#      gc(reset = TRUE)
+#      showMemoryUse(decreasing=TRUE, limit=5)
     }
     
     for (initYear in initYears) {

SCRIPTS/jobScriptReorder_sepYears_sepMembers_combLeadmonths

 #!/bin/bash
 #SBATCH --account=5120867-01
 #SBATCH --time=5000
-#SBATCH --mem=32024
+#SBATCH --mem=16024
 #SBATCH --ntasks=1
-#SBATCH --output=./log/log_doReorder_sepYears_sepMembers_combLeadmonths_<location>_initMonth<initMonth>_%j.txt
+#SBATCH --output=./log/log_doReorder_sepYears_sepMembers_combLeadmonths_<location>_<bcInfo>_<resolution>deg_initMonth<initMonth>_%j.txt
 #SBATCH --job-name=Reformat2
-#SBATCH --partition=ESG_Std
+#SBATCH --partition=ESG_High
 #SBATCH --mail-type=FAIL
 #SBATCH --mail-user=wietse.franssen@wur.nl
 # print date and time
@@ -17,6 +17,10 @@ module load R/3.2.0
 #
 # run current job
 echo '******** start Rscript...'
-Rscript ./tmpScripts/doReorder_sepYears_sepMembers_combLeadmonths_<location>_initMonth<initMonth>.R
+#ulimit -a
+#ulimit  -v 16000000  
+#ulimit -a
+
+Rscript ./tmpScripts/doReorder_sepYears_sepMembers_combLeadmonths_<location>_<bcInfo>_<resolution>deg_initMonth<initMonth>.R
 date