Commit c410a8cd authored by Franssen, Wietse's avatar Franssen, Wietse
Browse files

several improvements in 1b and step 2

parent 0a68f2a3
...@@ -88,7 +88,7 @@ for (variableName in variables) { ...@@ -88,7 +88,7 @@ for (variableName in variables) {
} }
## Apply mask ## Apply mask
for (iTime in 1:dim(RData$Data)[2]) { for (iTime in 1:length(RData$Dates$start)) {
RData$Data[iTime,,]<-RData$Data[iTime,,]*mask$Data[1,,] RData$Data[iTime,,]<-RData$Data[iTime,,]*mask$Data[1,,]
} }
......
#!/bin/bash #!/bin/bash
tmpScripts="./tmpScripts" tmpScripts="./tmpScripts"
#location="EU"; resolution="0.50" #location="EU"; resolution="0.50"
location="GHA"; resolution="0.50" #location="GHA"; resolution="0.50"
#location="GHA"; resolution="0.75" location="GHA"; resolution="0.75"
mkdir -p $tmpScripts mkdir -p $tmpScripts
......
...@@ -16,19 +16,21 @@ if (submitscript) { ...@@ -16,19 +16,21 @@ if (submitscript) {
initMonths <- c(X:X) initMonths <- c(X:X)
locName <- 'X' locName <- 'X'
resolution <- 'X' resolution <- 'X'
inPath <- sprintf("../DATA/System4_seasonal_15_rev1.0/%s_%s", locName, resolution) bcInfo <- 'X'
outPath <- sprintf("../DATA/System4_seasonal_15_rev1.1/%sdeg/%s_noBC", resolution, locName) inPath <- sprintf("../DATA/System4_seasonal_15_rev1.2/%sdeg/%s_%s", resolution, locName, bcInfo)
outPath <- sprintf("../DATA/System4_seasonal_15_rev1.2/%sdeg/%s_%s_biasformat", resolution, locName, bcInfo)
} else { } else {
members <- c(1:2) members <- c(1:2)
initYears <- c(1981:1982) initYears <- c(1981:1982)
initMonths <- 1 initMonths <- 1
#locName <- "GHA" locName <- "GHA"
locName <- "EU" #locName <- "EU"
resolution <- "0.75" resolution <- "0.75"
inPath <- sprintf("../DATA/System4_seasonal_15_rev1.0/%s_%s", locName, resolution) # bcInfo <- "BC"
outPath <- sprintf("../DATA/System4_seasonal_15_rev1.x/%sdeg/%s_noBC", resolution, locName) bcInfo <- "noBC"
inPath <- sprintf("../DATA/System4_seasonal_15_rev1.2/%sdeg/%s_%s", resolution, locName, bcInfo)
outPath <- sprintf("../DATA/System4_seasonal_15_rev1.2/%sdeg/%s_%s_biasformat", resolution, locName, bcInfo)
} }
mask<-Netcdf2R(inFile = sprintf("../DATA/mask/mask_wfdei_%s.nc4", locName),"mask")
variables<-names(variableInfo) variables<-names(variableInfo)
...@@ -39,14 +41,9 @@ leadMonths<-c(0:6) ...@@ -39,14 +41,9 @@ leadMonths<-c(0:6)
print("start") print("start")
for (initMonth in initMonths) { for (initMonth in initMonths) {
for (variableName in variables) { for (variableName in variables) {
if ('ecomsName' %in% names(variableInfo[[variableName]])) { inFile <- sprintf("%s/%s/%s_forcing_seas15_%s_%s_E<MEMBERS>_INIT<YEARS>_%02d.nc4", inPath, variableName, variableName, locName, bcInfo, initMonth)
variableNameECOMS<-variableInfo[[variableName]]$ecomsName
} else {
variableNameECOMS<-variableName
}
inFile <- sprintf("%s/%s/%s_forcing_seas15_%s_noBC_E<MEMBERS>_<YEARS>_%02d.RData", inPath, variableNameECOMS, variableNameECOMS, locName, initMonth)
print(inFile) print(inFile)
RDataAllLeadMonths <- reformat2Bias(members, initYears, initMonth, variableNameECOMS, locName, iFile = inFile) RDataAllLeadMonths <- reformat2Bias(members, initYears, initMonth, variableName, locName, iFile = inFile)
for (leadMonth in leadMonths) { for (leadMonth in leadMonths) {
targetMonth <- getInitTargetInfo( initYear = 0000, initMonth = initMonth, leadMonth = leadMonth )$targetMonth targetMonth <- getInitTargetInfo( initYear = 0000, initMonth = initMonth, leadMonth = leadMonth )$targetMonth
...@@ -54,8 +51,8 @@ for (initMonth in initMonths) { ...@@ -54,8 +51,8 @@ for (initMonth in initMonths) {
targetEYear <- getInitTargetInfo( initYear = initYears[length(initYears)], initMonth = initMonth, leadMonth = leadMonth )$targetYear targetEYear <- getInitTargetInfo( initYear = initYears[length(initYears)], initMonth = initMonth, leadMonth = leadMonth )$targetYear
print(sprintf("targetmonth: %s, leadMonth: %s",targetMonth, leadMonth)) print(sprintf("targetmonth: %s, leadMonth: %s",targetMonth, leadMonth))
oPrefix <- sprintf("%s/%s_forcing_seas15_%s_noBC_E%02d-%02d_TAR%4d-%4d_%02d_LM%d", oPrefix <- sprintf("%s/%s_forcing_seas15_%s_%s_E%02d-%02d_TAR%4d-%4d_%02d_LM%d",
outPath, variableName, locName, outPath, variableName, locName, bcInfo,
members[1], members[length(members)], members[1], members[length(members)],
targetSYear, targetEYear, targetSYear, targetEYear,
targetMonth, leadMonth) targetMonth, leadMonth)
...@@ -65,49 +62,6 @@ for (initMonth in initMonths) { ...@@ -65,49 +62,6 @@ for (initMonth in initMonths) {
# print(RData$InitializationDates) # print(RData$InitializationDates)
# print(RData$Dates$start) # print(RData$Dates$start)
## set all negative precipitation values 0
if (variableName == "pr") {
RData$Data[RData$Data < 0] <- 0
}
RData$xyCoords$x[]<-round(RData$xyCoords$x[],2)
RData$xyCoords$y[]<-round(RData$xyCoords$y[],2)
# ## check if grids are the same
# if (RData$xyCoords$x[1] != locationInfo$res0.50[[locName]]$lonmin ||
# RData$xyCoords$x[length(RData$xyCoords$x)] != locationInfo$res0.50[[locName]]$lonmax ||
# RData$xyCoords$y[1] != locationInfo$res0.50[[locName]]$latmin ||
# RData$xyCoords$y[length(RData$xyCoords$y)] != locationInfo$res0.50[[locName]]$latmax) {
#
# ## make grid
# newGrid<-getGrid(RData)
# newGrid$x <- c(locationInfo$res0.50[[locName]]$lonmin, locationInfo$res0.50[[locName]]$lonmax)
# newGrid$y <- c(locationInfo$res0.50[[locName]]$latmin, locationInfo$res0.50[[locName]]$latmax)
# attr(newGrid,"resX") <- 0.50
# attr(newGrid,"resY") <- 0.50
# ## regrid
# print(sprintf("Regridding..."))
# RData <- interpGridData(RData, new.grid = newGrid, method = "bilinear");
# }
## Apply mask
print("Apply mask")
for (iMember in 1:members) {
for (iTime in 1:dim(RData$Data)[2]) {
RData$Data[iMember,iTime,,]<-RData$Data[iMember,iTime,,]*mask$Data[1,,]
}
}
# Check units
RData$Variable$varName<-variableName
attr(RData$Variable,"standard_name")<-variableInfo[[variableName]]$standardName
attr(RData$Variable,"long_name")<-variableInfo[[variableName]]$longName
attr(RData$Variable,"units")<-variableInfo[[variableName]]$unitsEcoms
## add some extra attributes
attr(RData,"contact") <- "Wietse Franssen (wietse.franssen@wur.nl)"
#save(RData, file = paste0(oPrefix, ".RData")) #save(RData, file = paste0(oPrefix, ".RData"))
R2Netcdf(paste0(oPrefix, ".nc4"), RData) R2Netcdf(paste0(oPrefix, ".nc4"), RData)
} }
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
#SBATCH --time=5000 #SBATCH --time=5000
#SBATCH --mem=16024 #SBATCH --mem=16024
#SBATCH --ntasks=1 #SBATCH --ntasks=1
#SBATCH --output=./log/log_doReorder_combYears_combMembers_sepLeadmonths_<location>_<resolution>deg_initMonth<initMonth>_%j.txt #SBATCH --output=./log/log_2_doReorder_combYears_combMembers_sepLeadmonths_<location>_<bcInfo>_<resolution>deg_initMonth<initMonth>_%j.txt
#SBATCH --job-name=Reformat #SBATCH --job-name=Reformat
#SBATCH --partition=ESG_Std #SBATCH --partition=ESG_Std
#SBATCH --mail-type=FAIL #SBATCH --mail-type=FAIL
...@@ -17,6 +17,6 @@ module load R/3.2.0 ...@@ -17,6 +17,6 @@ module load R/3.2.0
# #
# run current job # run current job
echo '******** start Rscript...' echo '******** start Rscript...'
Rscript ./tmpScripts/doReorder_combYears_combMembers_sepLeadmonths_<location>_<resolution>deg_initMonth<initMonth>.R Rscript ./tmpScripts/2_doReorder_combYears_combMembers_sepLeadmonths_<location>_<bcInfo>_<resolution>deg_initMonth<initMonth>.R
date date
...@@ -4,6 +4,9 @@ tmpScripts="./tmpScripts" ...@@ -4,6 +4,9 @@ tmpScripts="./tmpScripts"
#location="GHA"; resolution="0.50" #location="GHA"; resolution="0.50"
location="GHA"; resolution="0.75" location="GHA"; resolution="0.75"
bcInfo="noBC"
#bcInfo="BC"
mkdir -p $tmpScripts mkdir -p $tmpScripts
for iInitMonth in $(seq -f "%02g" 1 12); do for iInitMonth in $(seq -f "%02g" 1 12); do
...@@ -11,11 +14,13 @@ for iInitMonth in $(seq -f "%02g" 1 12); do ...@@ -11,11 +14,13 @@ for iInitMonth in $(seq -f "%02g" 1 12); do
sed -e "s|<initMonth>|$iInitMonth|g" \ sed -e "s|<initMonth>|$iInitMonth|g" \
-e "s|<location>|$location|g" \ -e "s|<location>|$location|g" \
-e "s|<resolution>|$resolution|g" \ -e "s|<resolution>|$resolution|g" \
jobScriptReorder_combYears_combMembers_sepLeadmonths > $tmpScripts"/jobScriptReorder_combYears_combMembers_sepLeadmonths_"$location"_"$resolution"deg_initMonth"$iInitMonth -e "s|<bcInfo>|$bcInfo|g" \
2_jobScriptReorder_combYears_combMembers_sepLeadmonths > $tmpScripts"/2_jobScriptReorder_combYears_combMembers_sepLeadmonths_"$location"_"$bcInfo"_"$resolution"deg_initMonth"$iInitMonth
sed -e "s|submitscript <- FALSE|submitscript <- TRUE|g" \ sed -e "s|submitscript <- FALSE|submitscript <- TRUE|g" \
-e "s|initMonths <- c(X:X)|initMonths <- c($iInitMonth:$iInitMonth)|g" \ -e "s|initMonths <- c(X:X)|initMonths <- c($iInitMonth:$iInitMonth)|g" \
-e "s|locName <- 'X'|locName <- '$location'|g" \ -e "s|locName <- 'X'|locName <- '$location'|g" \
-e "s|resolution <- 'X'|resolution <- '$resolution'|g" \ -e "s|resolution <- 'X'|resolution <- '$resolution'|g" \
doReorder_combYears_combMembers_sepLeadmonths.R > $tmpScripts"/doReorder_combYears_combMembers_sepLeadmonths_"$location"_"$resolution"deg_initMonth"$iInitMonth".R" -e "s|bcInfo <- 'X'|bcInfo <- '$bcInfo'|g" \
sbatch "$tmpScripts/jobScriptReorder_combYears_combMembers_sepLeadmonths_"$location"_"$resolution"deg_initMonth"$iInitMonth 2_doReorder_combYears_combMembers_sepLeadmonths.R > $tmpScripts"/2_doReorder_combYears_combMembers_sepLeadmonths_"$location"_"$bcInfo"_"$resolution"deg_initMonth"$iInitMonth".R"
sbatch "$tmpScripts/2_jobScriptReorder_combYears_combMembers_sepLeadmonths_"$location"_"$bcInfo"_"$resolution"deg_initMonth"$iInitMonth
done done
rm(list=ls()) rm(list=ls())
source(file = "./functions/functionsGeneral.R") source(file = "./functions/functionsGeneral.R")
source(file = "./functions/functionNetcdf2R.R")
reformat2Bias<-function(members, initYears, initMonth, varName, locName, reformat2Bias<-function(members, initYears, initMonth, varName, locName,
iFile = "./forcing_seas15_GHA_ref__E<MEMBERS>_<YEARS>_01.RData") { iFile = "./forcing_seas15_GHA_ref__E<MEMBERS>_<YEARS>_01.nc4") {
sYear<-initYears[1] sYear<-initYears[1]
eYear<-initYears[length(initYears)] eYear<-initYears[length(initYears)]
nYears<-length(initYears) nYears<-length(initYears)
...@@ -24,16 +25,16 @@ reformat2Bias<-function(members, initYears, initMonth, varName, locName, ...@@ -24,16 +25,16 @@ reformat2Bias<-function(members, initYears, initMonth, varName, locName,
indexesYearsLeadMonths<-indexesForYearsPerLeadMonth(sYear,eYear,initMonth) indexesYearsLeadMonths<-indexesForYearsPerLeadMonth(sYear,eYear,initMonth)
## Change to proper filename ## Change to proper filename
nameFileR <- iFile nameFile <- iFile
strYear <- sprintf("%4d",initYear) strYear <- sprintf("%4d",initYear)
strMember <- sprintf("%02d",iMember) strMember <- sprintf("%02d",iMember)
nameFileR <-gsub("<MEMBERS>", strMember, nameFileR) nameFile <-gsub("<MEMBERS>", strMember, nameFile)
nameFileR <-gsub("<YEARS>", strYear, nameFileR) nameFile <-gsub("<YEARS>", strYear, nameFile)
print(paste0("Opening: ", nameFileR)) print(paste0("Opening: ", nameFile))
load(nameFileR) RData<-Netcdf2R(nameFile, varName)
if (sMember == iMember) { if (sMember == iMember) {
strInitDates<-c(strInitDates, sys4$InitializationDates) strInitDates<-c(strInitDates, RData$InitializationDates)
} }
if (!allocatedOutputArray) { if (!allocatedOutputArray) {
...@@ -42,7 +43,7 @@ reformat2Bias<-function(members, initYears, initMonth, varName, locName, ...@@ -42,7 +43,7 @@ reformat2Bias<-function(members, initYears, initMonth, varName, locName,
nYears<-eYear-sYear+1 nYears<-eYear-sYear+1
## allocate list ## allocate list
final<-sys4 final<-RData
## allocate arrays ## allocate arrays
finalList<-NULL finalList<-NULL
listNames<-NULL listNames<-NULL
...@@ -51,7 +52,7 @@ reformat2Bias<-function(members, initYears, initMonth, varName, locName, ...@@ -51,7 +52,7 @@ reformat2Bias<-function(members, initYears, initMonth, varName, locName,
listNames<-c(listNames,paste0("LeadMonth_",iLM-1)) listNames<-c(listNames,paste0("LeadMonth_",iLM-1))
totalDays<-indexesYearsLeadMonths$eIndexes[as.character(eYear),iLM] totalDays<-indexesYearsLeadMonths$eIndexes[as.character(eYear),iLM]
finalList[[iLM]]$Data<-array(0,dim=c(nMembers,totalDays,length(sys4$xyCoords$y),length(sys4$xyCoords$x))) finalList[[iLM]]$Data<-array(0,dim=c(nMembers,totalDays,length(RData$xyCoords$y),length(RData$xyCoords$x)))
finalList[[iLM]]$Dates$start<-character(length=totalDays) finalList[[iLM]]$Dates$start<-character(length=totalDays)
finalList[[iLM]]$Dates$end<-character(length=totalDays) finalList[[iLM]]$Dates$end<-character(length=totalDays)
attr(finalList[[iLM]]$Data,"dimensions") <- c("member","time","lat","lon") attr(finalList[[iLM]]$Data,"dimensions") <- c("member","time","lat","lon")
...@@ -70,9 +71,9 @@ reformat2Bias<-function(members, initYears, initMonth, varName, locName, ...@@ -70,9 +71,9 @@ reformat2Bias<-function(members, initYears, initMonth, varName, locName,
## Split the data in seperate months ## Split the data in seperate months
parts<-NULL parts<-NULL
for (iLM in 1:7) { for (iLM in 1:7) {
parts[[iLM]] <- list(startDates = sys4$Dates$start[indexes[iLM,"sIndex"]:indexes[iLM,"eIndex"]], parts[[iLM]] <- list(startDates = RData$Dates$start[indexes[iLM,"sIndex"]:indexes[iLM,"eIndex"]],
endDates = sys4$Dates$end[indexes[iLM,"sIndex"]:indexes[iLM,"eIndex"]], endDates = RData$Dates$end[indexes[iLM,"sIndex"]:indexes[iLM,"eIndex"]],
data = sys4$Data[indexes[iLM,"sIndex"]:indexes[iLM,"eIndex"],,]) data = RData$Data[indexes[iLM,"sIndex"]:indexes[iLM,"eIndex"],,])
} }
## Fill big data array ## Fill big data array
...@@ -84,7 +85,7 @@ reformat2Bias<-function(members, initYears, initMonth, varName, locName, ...@@ -84,7 +85,7 @@ reformat2Bias<-function(members, initYears, initMonth, varName, locName,
finalList[[iLM]]$Dates$start[sIndex:eIndex] <-parts[[iLM]]$startDates finalList[[iLM]]$Dates$start[sIndex:eIndex] <-parts[[iLM]]$startDates
finalList[[iLM]]$Dates$end[sIndex:eIndex] <-parts[[iLM]]$endDates finalList[[iLM]]$Dates$end[sIndex:eIndex] <-parts[[iLM]]$endDates
} }
initDates[iInitYear]<-sys4$Dates$start[1] initDates[iInitYear]<-RData$Dates$start[1]
iInitYear<-iInitYear+1 iInitYear<-iInitYear+1
} }
......
...@@ -14,7 +14,7 @@ variableInfo <- list( ...@@ -14,7 +14,7 @@ variableInfo <- list(
tasmin = list(standardName = "air_min_temperature", longName = "Daily Minimum Near-Surface Air Temperature", unitsEcoms = "Celsius", units = "K"), tasmin = list(standardName = "air_min_temperature", longName = "Daily Minimum Near-Surface Air Temperature", unitsEcoms = "Celsius", units = "K"),
tasmax = list(standardName = "air_max_temperature", longName = "Daily Maximum Near-Surface Air Temperature", unitsEcoms = "Celsius", units = "K"), tasmax = list(standardName = "air_max_temperature", longName = "Daily Maximum Near-Surface Air Temperature", unitsEcoms = "Celsius", units = "K"),
pr = list(standardName = "precipitation_flux", longName = "Precipitation", unitsEcoms = "mm day-1", units = "kg m-2 s-1", ecomsName = "tp"), pr = list(standardName = "precipitation_flux", longName = "Precipitation", unitsEcoms = "mm day-1", units = "kg m-2 s-1", ecomsName = "tp"),
psl = list(standardName = "air_pressure_at_sea_level", longName = "Sea level pressure", unitsEcoms = "Pa", units = "Pa"), # psl = list(standardName = "air_pressure_at_sea_level", longName = "Sea level pressure", unitsEcoms = "Pa", units = "Pa"),
rsds = list(standardName = "surface_downwelling_shortwave_flux_in_air", longName = "Surface Downwelling Shortwave Radiation", unitsEcoms = "1/86400^2 W m-2", units = "W m-2"), rsds = list(standardName = "surface_downwelling_shortwave_flux_in_air", longName = "Surface Downwelling Shortwave Radiation", unitsEcoms = "1/86400^2 W m-2", units = "W m-2"),
rlds = list(standardName = "surface_downwelling_longwave_flux_in_air", longName = "Surface Downwelling Longwave Radiation", unitsEcoms = "1/86400^2 W m-2", units = "W m-2"), rlds = list(standardName = "surface_downwelling_longwave_flux_in_air", longName = "Surface Downwelling Longwave Radiation", unitsEcoms = "1/86400^2 W m-2", units = "W m-2"),
huss = list(standardName = "specific_humidity", longName = "Near-Surface Specific Humidity", unitsEcoms = "1", units = "1"), huss = list(standardName = "specific_humidity", longName = "Near-Surface Specific Humidity", unitsEcoms = "1", units = "1"),
...@@ -26,7 +26,7 @@ colorbarLimits <- list( ...@@ -26,7 +26,7 @@ colorbarLimits <- list(
tasmin = list(minValue = -20, maxValue = 35, diffMin = -0.5, diffMax = 0.5), tasmin = list(minValue = -20, maxValue = 35, diffMin = -0.5, diffMax = 0.5),
tasmax = list(minValue = -20, maxValue = 35, diffMin = -0.5, diffMax = 0.5), tasmax = list(minValue = -20, maxValue = 35, diffMin = -0.5, diffMax = 0.5),
pr = list(minValue = 0, maxValue = 10, diffMin = -0.2, diffMax = 0.2), pr = list(minValue = 0, maxValue = 10, diffMin = -0.2, diffMax = 0.2),
psl = list(minValue = 0, maxValue = 10, diffMin = -0.2, diffMax = 0.2), # psl = list(minValue = 0, maxValue = 10, diffMin = -0.2, diffMax = 0.2),
rsds = list(minValue = -200, maxValue = 500, diffMin = -0.2, diffMax = 0.2), rsds = list(minValue = -200, maxValue = 500, diffMin = -0.2, diffMax = 0.2),
rlds = list(minValue = -200, maxValue = 500, diffMin = -0.2, diffMax = 0.2), rlds = list(minValue = -200, maxValue = 500, diffMin = -0.2, diffMax = 0.2),
huss = list(minValue = 0, maxValue = 10, diffMin = -0.2, diffMax = 0.2), huss = list(minValue = 0, maxValue = 10, diffMin = -0.2, diffMax = 0.2),
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment