From 014bec50fb8f2222820e087da70c09553db8243b Mon Sep 17 00:00:00 2001
From: Arne Babenhauserheide <amvdw95@gmail.com>
Date: Wed, 15 Apr 2015 15:06:52 +0000
Subject: [PATCH] undid savedict rewriting to function call, because that is
unmergeable.
---
refactored/da/analysis/expand_fluxes.py | 279 +++++++++++++++---------
1 file changed, 171 insertions(+), 108 deletions(-)
diff --git a/refactored/da/analysis/expand_fluxes.py b/refactored/da/analysis/expand_fluxes.py
index 8b635a1..02e50e2 100755
--- a/refactored/da/analysis/expand_fluxes.py
+++ b/refactored/da/analysis/expand_fluxes.py
@@ -50,29 +50,6 @@ def _getpriordata(DaCycle, StateVector, startdate, enddate, nlag, cyclelength):
raise ValueError("No savestate.nc found in the possible paths:" + str(tried) + ". Likely the timeframe is no multiple of the cycle length: " + str(cyclelength) + ". startdate: " + str(startdate) + " enddate: " + str(enddate))
-def _saveInto(ncf, standardvar, data, dims,
- count=None, name=None, long_name=None,
- units=None, comment=None, nsets=None,
- standard_name=None):
- """save the given data into the netcdf file using one of the StandardVars in tools/io4.py as template.
-
- :param standardvar: The name of the standard definition to use.
- :param count: offset for writing the data (==previous data which should not be replaced)."""
- savedict=ncf.StandardVar(varname=standardvar)
- savedict["values"] = data
- savedict["dims"] = dims
- for key, val in [("count", count),
- ("name", name),
- ("long_name", long_name),
- ("units", units),
- ("standard_name", standard_name),
- ("comment", comment)]:
- if val: savedict[key] = val
- if nsets:
- ncf.AddData(savedict, nsets=nsets)
- return ncf.AddData(savedict)
-
-
def SaveWeeklyAvg1x1Data(DaCycle, StateVector):
"""
Function creates a NetCDF file with output on 1x1 degree grid. It uses the flux data written by the
@@ -174,24 +151,49 @@ def SaveWeeklyAvg1x1Data(DaCycle, StateVector):
#
# For each dataset, get the standard definitions from the module mysettings, add values, dimensions, and unlimited count, then write
#
-
- for name, data in [('bio_flux_'+qual_short, biomapped),
- ('ocn_flux_'+qual_short, oceanmapped),
- ('bio_flux_%s_cov'%qual_short, biovarmapped),
- ('ocn_flux_%s_cov'%qual_short, oceanvarmapped)]:
- _saveInto(ncf, name, data.tolist(),
- dims=dimdate+dimgrid, count=next)
-
+ savedict=ncf.StandardVar(varname='bio_flux_'+qual_short)
+ savedict['values']=biomapped.tolist()
+ savedict['dims']=dimdate+dimgrid
+ savedict['count']=next
+ ncf.AddData(savedict)
+#
+ savedict=ncf.StandardVar(varname='ocn_flux_'+qual_short)
+ savedict['values']=oceanmapped.tolist()
+ savedict['dims']=dimdate+dimgrid
+ savedict['count']=next
+ ncf.AddData(savedict)
+
+ savedict=ncf.StandardVar(varname='bio_flux_%s_cov'%qual_short)
+ savedict['values']=biovarmapped.tolist()
+ savedict['dims']=dimdate+dimgrid
+ savedict['count']=next
+ ncf.AddData(savedict)
+#
+ savedict=ncf.StandardVar(varname='ocn_flux_%s_cov'%qual_short)
+ savedict['values']=oceanvarmapped.tolist()
+ savedict['dims']=dimdate+dimgrid
+ savedict['count']=next
+ ncf.AddData(savedict)
# End prior/posterior block
- for name, data in [('fire_flux_imp', fire),
- ('fossil_flux_imp', fossil)]:
- _saveInto(ncf, name, data.tolist(),
- dims=dimdate+dimgrid, count=next)
-
- _saveInto(ncf, 'date', date2num(startdate)-dectime0+dt.days/2.0,
- dims=dimdate, count=next)
+ savedict=ncf.StandardVar(varname='fire_flux_imp')
+ savedict['values']=fire.tolist()
+ savedict['dims']=dimdate+dimgrid
+ savedict['count']=next
+ ncf.AddData(savedict)
+#
+ savedict=ncf.StandardVar(varname='fossil_flux_imp')
+ savedict['values']=fossil.tolist()
+ savedict['dims']=dimdate+dimgrid
+ savedict['count']=next
+ ncf.AddData(savedict)
+#
+ savedict=ncf.StandardVar(varname='date')
+ savedict['values']=date2num(startdate)-dectime0+dt.days/2.0
+ savedict['dims']=dimdate
+ savedict['count']=next
+ ncf.AddData(savedict)
sys.stdout.write('.')
sys.stdout.flush()
@@ -299,7 +301,16 @@ def SaveWeeklyAvgStateData(DaCycle, StateVector):
if prior:
qual_short='prior'
- dummy, dummy, choicelag = _getpriordata(DaCycle, StateVector, startdate, enddate, nlag, cyclelength=dt.days)
+ for n in range(nlag,0,-1):
+ priordate = enddate - timedelta(dt.days*n)
+ savedir = DaCycle['dir.output'].replace(startdate.strftime('%Y%m%d'),priordate.strftime('%Y%m%d') )
+ filename = os.path.join(savedir,'savestate.nc')
+ if os.path.exists(filename):
+ dummy = StateVector.ReadFromFile(filename)
+ choicelag = n
+
+ msg = 'Read prior dataset from file %s, sds %d: '%(filename,n) ; logging.debug(msg)
+ break
else:
qual_short='opt'
savedir = DaCycle['dir.output']
@@ -313,50 +324,83 @@ def SaveWeeklyAvgStateData(DaCycle, StateVector):
#
data = StateVector.EnsembleMembers[choicelag-1][0].ParameterValues*vectorbio # units of mole region-1 s-1
- _saveInto(ncf, 'bio_flux_%s'%qual_short, data,
- dims=dimdate+dimregs, count=next)
+ savedict = ncf.StandardVar(varname='bio_flux_%s'%qual_short)
+ savedict['values'] = data
+ savedict['dims'] = dimdate+dimregs
+ savedict['count'] = next
+ ncf.AddData(savedict)
members = StateVector.EnsembleMembers[choicelag-1]
deviations = np.array([mem.ParameterValues*data for mem in members])
- _saveInto(ncf, 'bio_flux_%s_cov'%qual_short, deviations.tolist(),
- dims=dimdate+dimmembers+dimregs, count=next,
- units="mol region-1 s-1",
- comment="This is the matrix square root, use (M x M^T)/(nmembers-1) to make covariance")
-
- _saveInto(ncf, 'unknown', deviations.std(axis=0),
- dims=dimdate+dimregs, count=next,
- name = 'bio_flux_%s_std'%qual_short,
- long_name = 'Biosphere flux standard deviation, %s'%qual_short,
- comment = "This is the standard deviation on each parameter",
- units = "mol region-1 s-1")
+ savedict=ncf.StandardVar(varname='bio_flux_%s_cov'%qual_short)
+
+ savedict['values']=deviations.tolist()
+ savedict['dims']=dimdate+dimmembers+dimregs
+ savedict['comment']="This is the matrix square root, use (M x M^T)/(nmembers-1) to make covariance"
+ savedict['units']="mol region-1 s-1"
+ savedict['count']=next
+ ncf.AddData(savedict)
+
+ savedict=ncf.StandardVar('unknown')
+ savedict['name'] = 'bio_flux_%s_std'%qual_short
+ savedict['long_name'] = 'Biosphere flux standard deviation, %s'%qual_short
+ savedict['values']=deviations.std(axis=0)
+ savedict['dims']=dimdate+dimregs
+ savedict['comment']="This is the standard deviation on each parameter"
+ savedict['units']="mol region-1 s-1"
+ savedict['count']=next
+ ncf.AddData(savedict)
data = StateVector.EnsembleMembers[choicelag-1][0].ParameterValues*vectorocn # units of mole region-1 s-1
- _saveInto(ncf, 'ocn_flux_%s'%qual_short, data,
- dims=dimdate+dimregs, count=next)
+ savedict=ncf.StandardVar(varname='ocn_flux_%s'%qual_short)
+ savedict['values']=data
+ savedict['dims']=dimdate+dimregs
+ savedict['count']=next
+ ncf.AddData(savedict)
deviations = np.array([mem.ParameterValues*data for mem in members])
- _saveInto(ncf, 'ocn_flux_%s_cov'%qual_short, deviations.tolist(),
- dims=dimdate+dimmembers+dimregs, count=next,
- comment="This is the matrix square root, use (M x M^T)/(nmembers-1) to make covariance",
- units="mol region-1 s-1")
-
- _saveInto(ncf, 'unknown', deviations.std(axis=0),
- dims=dimdate+dimregs, count=next,
- name = 'ocn_flux_%s_std'%qual_short,
- long_name = 'Ocean flux standard deviation, %s'%qual_short,
- comment = "This is the standard deviation on each parameter",
- units = "mol region-1 s-1")
-
- for name, data in [('fire_flux_imp', vectorfire),
- ('fossil_flux_imp', vectorfossil)]:
- _saveInto(ncf, name, data,
- dims=dimdate+dimregs, count=next)
-
- _saveInto(ncf, 'date', ncfdate,
- dims=dimdate, count=next)
+ savedict=ncf.StandardVar(varname='ocn_flux_%s_cov'%qual_short)
+ savedict['values']=deviations.tolist()
+ savedict['dims']=dimdate+dimmembers+dimregs
+ savedict['comment']="This is the matrix square root, use (M x M^T)/(nmembers-1) to make covariance"
+ savedict['units']="mol region-1 s-1"
+ savedict['count']=next
+ ncf.AddData(savedict)
+
+ savedict=ncf.StandardVar('unknown')
+ savedict['name'] = 'ocn_flux_%s_std'%qual_short
+ savedict['long_name'] = 'Ocean flux standard deviation, %s'%qual_short
+ savedict['values']=deviations.std(axis=0)
+ savedict['dims']=dimdate+dimregs
+ savedict['comment']="This is the standard deviation on each parameter"
+ savedict['units']="mol region-1 s-1"
+ savedict['count']=next
+ ncf.AddData(savedict)
+
+ data = vectorfire
+
+ savedict=ncf.StandardVar(varname='fire_flux_imp')
+ savedict['values']=data
+ savedict['dims']=dimdate+dimregs
+ savedict['count']=next
+ ncf.AddData(savedict)
+
+ data = vectorfossil
+
+ savedict=ncf.StandardVar(varname='fossil_flux_imp')
+ savedict['values']=data
+ savedict['dims']=dimdate+dimregs
+ savedict['count']=next
+ ncf.AddData(savedict)
+
+ savedict=ncf.StandardVar(varname='date')
+ savedict['values'] = ncfdate
+ savedict['dims'] = dimdate
+ savedict['count'] = next
+ ncf.AddData(savedict)
sys.stdout.write('.')
sys.stdout.flush()
@@ -456,8 +500,11 @@ def SaveWeeklyAvgTCData(DaCycle, StateVector):
# First add the date for this cycle to the file, this grows the unlimited dimension
- _saveInto(ncf, 'date', ncfdate,
- dims=dimdate, count=index)
+ savedict = ncf.StandardVar(varname='date')
+ savedict['values'] = ncfdate
+ savedict['dims'] = dimdate
+ savedict['count'] = index
+ dummy = ncf.AddData(savedict)
# Now convert other variables that were inside the flux_1x1 file
@@ -466,24 +513,31 @@ def SaveWeeklyAvgTCData(DaCycle, StateVector):
data = ncf_in.GetVariable(vname)[index]
+ savedict = ncf.StandardVar(varname=vname)
+
if vname=='latitude': continue
elif vname=='longitude': continue
elif vname=='date': continue
elif vname=='idate': continue
elif 'cov' in vname:
+
#AB? Only works with recent version of numpy, not with 1.4.1 installed on the suns
#cov = data.transpose().dot(data)
cov = np.dot(data.transpose(),data)
tcdata = StateVector.VectorToTC(vectordata=cov,cov=True) # vector to TC
- units = '[mol/region/s]**2'
- dims = dimdate+dimregs+dimregs
+
+ savedict['units'] = '[mol/region/s]**2'
+ savedict['dims'] = dimdate+dimregs+dimregs
+
else:
+
tcdata = StateVector.VectorToTC(vectordata=data) # vector to TC
- dims = dimdate+dimregs
- units = 'mol/region/s'
- _saveInto(ncf, vname, tcdata,
- dims=dims, units=units)
+ savedict['dims'] = dimdate+dimregs
+ savedict['units'] = 'mol/region/s'
+
+ savedict['values'] = tcdata
+ ncf.AddData(savedict)
ncf_in.close()
ncf.close()
@@ -545,10 +599,12 @@ def SaveTCDataExt(rundat):
dims=dimdate+dimregs
print vname,data.shape
- _saveInto(ncf, vname, data.tolist(),
- dims=dims, count=0,
- units='mol/region/s',
- nsets=data.shape[0])
+ savedict = ncf.StandardVar(varname=vname)
+ savedict['values'] = data.tolist()
+ savedict['dims'] = dims
+ savedict['units'] = 'mol/region/s'
+ savedict['count'] = 0
+ ncf.AddData(savedict,nsets=data.shape[0])
ncf.close()
@@ -601,10 +657,11 @@ def SaveEcoDataExt(rundat):
else:
print 'Dataset with unknown dimensions encountered in file: %s'%vname
- _saveInto(ncf, vname, data.tolist(),
- dims=dims,
- units=atts['units'],
- nsets=data.shape[0])
+ savedict=ncf.StandardVar(varname=vname)
+ savedict['units']=atts['units']
+ savedict['values']=data.tolist()
+ savedict['dims']=dims
+ ncf.AddData(savedict,nsets=data.shape[0])
ncf.close()
@@ -695,16 +752,18 @@ def SaveTimeAvgData(rundat,infile,avg='monthly'):
if ncf.StandardVar(sds)['name'] == 'unknown':
- invar = file.variables[sds]
- _saveInto(ncf, sds, data.tolist(),
- dims=dims, count=0,
- name=sds,
- units=invar.units,
- long_name=invar.long_name,
- comment=invar.comment,
- standard_name=invar.standard_name)
- sys.stdout.write('.')
- sys.stdout.flush()
+ savedict = ncf.StandardVar(sds)
+ savedict['name'] = sds
+ savedict['values'] = data.tolist()
+ savedict['dims'] = dims
+ savedict['units'] = file.variables[sds].units
+ savedict['long_name'] = file.variables[sds].long_name
+ savedict['comment'] = file.variables[sds].comment
+ savedict['standard_name'] = file.variables[sds].standard_name
+ savedict['count'] =0
+ ncf.AddData(savedict)
+ sys.stdout.write('.')
+ sys.stdout.flush()
else:
@@ -728,18 +787,22 @@ def SaveTimeAvgData(rundat,infile,avg='monthly'):
count=count+1
if not ncf.has_date(date2num(dd)-dectime0):
- _saveInto(ncf, 'date', date2num(dd)-dectime0,
- dims=dimdate, count=count)
- units=file.variables[sds].units
+ savedict=ncf.StandardVar('date')
+ savedict['values']=date2num(dd)-dectime0
+ savedict['dims']=dimdate
+ savedict['count']=count
+ ncf.AddData(savedict)
+
+ savedict=ncf.StandardVar(sds)
+ savedict['values']=data.tolist()
+ savedict['units']=file.variables[sds].units
if 'cov' in sds:
- dimstmp=dimdate+dims+dims
+ savedict['dims']=dimdate+dims+dims
else:
- dimstmp=dimdate+dims
-
- _saveInto(ncf, sds, data.tolist(),
- dims=dimstmp, count=count,
- units=units)
+ savedict['dims']=dimdate+dims
+ savedict['count']=count
+ ncf.AddData(savedict)
sys.stdout.write('.')
sys.stdout.flush()
--
GitLab