Skip to content
Snippets Groups Projects
Commit 014bec50 authored by Arne Babenhauserheide's avatar Arne Babenhauserheide
Browse files

undid savedict rewriting to function call, because that is unmergeable.

parent f3143e13
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
refactored/da/analysis/expand_fluxes.py
+ 171
− 108
View file @ 014bec50
...@@ -50,29 +50,6 @@ def _getpriordata(DaCycle, StateVector, startdate, enddate, nlag, cyclelength): ...@@ -50,29 +50,6 @@ def _getpriordata(DaCycle, StateVector, startdate, enddate, nlag, cyclelength):
raise ValueError("No savestate.nc found in the possible paths:" + str(tried) + ". Likely the timeframe is no multiple of the cycle length: " + str(cyclelength) + ". startdate: " + str(startdate) + " enddate: " + str(enddate)) raise ValueError("No savestate.nc found in the possible paths:" + str(tried) + ". Likely the timeframe is no multiple of the cycle length: " + str(cyclelength) + ". startdate: " + str(startdate) + " enddate: " + str(enddate))
def _saveInto(ncf, standardvar, data, dims,
count=None, name=None, long_name=None,
units=None, comment=None, nsets=None,
standard_name=None):
"""save the given data into the netcdf file using one of the StandardVars in tools/io4.py as template.
:param standardvar: The name of the standard definition to use.
:param count: offset for writing the data (==previous data which should not be replaced)."""
savedict=ncf.StandardVar(varname=standardvar)
savedict["values"] = data
savedict["dims"] = dims
for key, val in [("count", count),
("name", name),
("long_name", long_name),
("units", units),
("standard_name", standard_name),
("comment", comment)]:
if val: savedict[key] = val
if nsets:
ncf.AddData(savedict, nsets=nsets)
return ncf.AddData(savedict)
def SaveWeeklyAvg1x1Data(DaCycle, StateVector): def SaveWeeklyAvg1x1Data(DaCycle, StateVector):
""" """
Function creates a NetCDF file with output on 1x1 degree grid. It uses the flux data written by the Function creates a NetCDF file with output on 1x1 degree grid. It uses the flux data written by the
...@@ -174,24 +151,49 @@ def SaveWeeklyAvg1x1Data(DaCycle, StateVector): ...@@ -174,24 +151,49 @@ def SaveWeeklyAvg1x1Data(DaCycle, StateVector):
# #
# For each dataset, get the standard definitions from the module mysettings, add values, dimensions, and unlimited count, then write # For each dataset, get the standard definitions from the module mysettings, add values, dimensions, and unlimited count, then write
# #
savedict=ncf.StandardVar(varname='bio_flux_'+qual_short)
for name, data in [('bio_flux_'+qual_short, biomapped), savedict['values']=biomapped.tolist()
('ocn_flux_'+qual_short, oceanmapped), savedict['dims']=dimdate+dimgrid
('bio_flux_%s_cov'%qual_short, biovarmapped), savedict['count']=next
('ocn_flux_%s_cov'%qual_short, oceanvarmapped)]: ncf.AddData(savedict)
_saveInto(ncf, name, data.tolist(), #
dims=dimdate+dimgrid, count=next) savedict=ncf.StandardVar(varname='ocn_flux_'+qual_short)
savedict['values']=oceanmapped.tolist()
savedict['dims']=dimdate+dimgrid
savedict['count']=next
ncf.AddData(savedict)
savedict=ncf.StandardVar(varname='bio_flux_%s_cov'%qual_short)
savedict['values']=biovarmapped.tolist()
savedict['dims']=dimdate+dimgrid
savedict['count']=next
ncf.AddData(savedict)
#
savedict=ncf.StandardVar(varname='ocn_flux_%s_cov'%qual_short)
savedict['values']=oceanvarmapped.tolist()
savedict['dims']=dimdate+dimgrid
savedict['count']=next
ncf.AddData(savedict)
# End prior/posterior block # End prior/posterior block
for name, data in [('fire_flux_imp', fire), savedict=ncf.StandardVar(varname='fire_flux_imp')
('fossil_flux_imp', fossil)]: savedict['values']=fire.tolist()
_saveInto(ncf, name, data.tolist(), savedict['dims']=dimdate+dimgrid
dims=dimdate+dimgrid, count=next) savedict['count']=next
ncf.AddData(savedict)
_saveInto(ncf, 'date', date2num(startdate)-dectime0+dt.days/2.0, #
dims=dimdate, count=next) savedict=ncf.StandardVar(varname='fossil_flux_imp')
savedict['values']=fossil.tolist()
savedict['dims']=dimdate+dimgrid
savedict['count']=next
ncf.AddData(savedict)
#
savedict=ncf.StandardVar(varname='date')
savedict['values']=date2num(startdate)-dectime0+dt.days/2.0
savedict['dims']=dimdate
savedict['count']=next
ncf.AddData(savedict)
sys.stdout.write('.') sys.stdout.write('.')
sys.stdout.flush() sys.stdout.flush()
...@@ -299,7 +301,16 @@ def SaveWeeklyAvgStateData(DaCycle, StateVector): ...@@ -299,7 +301,16 @@ def SaveWeeklyAvgStateData(DaCycle, StateVector):
if prior: if prior:
qual_short='prior' qual_short='prior'
dummy, dummy, choicelag = _getpriordata(DaCycle, StateVector, startdate, enddate, nlag, cyclelength=dt.days) for n in range(nlag,0,-1):
priordate = enddate - timedelta(dt.days*n)
savedir = DaCycle['dir.output'].replace(startdate.strftime('%Y%m%d'),priordate.strftime('%Y%m%d') )
filename = os.path.join(savedir,'savestate.nc')
if os.path.exists(filename):
dummy = StateVector.ReadFromFile(filename)
choicelag = n
msg = 'Read prior dataset from file %s, sds %d: '%(filename,n) ; logging.debug(msg)
break
else: else:
qual_short='opt' qual_short='opt'
savedir = DaCycle['dir.output'] savedir = DaCycle['dir.output']
...@@ -313,50 +324,83 @@ def SaveWeeklyAvgStateData(DaCycle, StateVector): ...@@ -313,50 +324,83 @@ def SaveWeeklyAvgStateData(DaCycle, StateVector):
# #
data = StateVector.EnsembleMembers[choicelag-1][0].ParameterValues*vectorbio # units of mole region-1 s-1 data = StateVector.EnsembleMembers[choicelag-1][0].ParameterValues*vectorbio # units of mole region-1 s-1
_saveInto(ncf, 'bio_flux_%s'%qual_short, data, savedict = ncf.StandardVar(varname='bio_flux_%s'%qual_short)
dims=dimdate+dimregs, count=next) savedict['values'] = data
savedict['dims'] = dimdate+dimregs
savedict['count'] = next
ncf.AddData(savedict)
members = StateVector.EnsembleMembers[choicelag-1] members = StateVector.EnsembleMembers[choicelag-1]
deviations = np.array([mem.ParameterValues*data for mem in members]) deviations = np.array([mem.ParameterValues*data for mem in members])
_saveInto(ncf, 'bio_flux_%s_cov'%qual_short, deviations.tolist(), savedict=ncf.StandardVar(varname='bio_flux_%s_cov'%qual_short)
dims=dimdate+dimmembers+dimregs, count=next,
units="mol region-1 s-1", savedict['values']=deviations.tolist()
comment="This is the matrix square root, use (M x M^T)/(nmembers-1) to make covariance") savedict['dims']=dimdate+dimmembers+dimregs
savedict['comment']="This is the matrix square root, use (M x M^T)/(nmembers-1) to make covariance"
_saveInto(ncf, 'unknown', deviations.std(axis=0), savedict['units']="mol region-1 s-1"
dims=dimdate+dimregs, count=next, savedict['count']=next
name = 'bio_flux_%s_std'%qual_short, ncf.AddData(savedict)
long_name = 'Biosphere flux standard deviation, %s'%qual_short,
comment = "This is the standard deviation on each parameter", savedict=ncf.StandardVar('unknown')
units = "mol region-1 s-1") savedict['name'] = 'bio_flux_%s_std'%qual_short
savedict['long_name'] = 'Biosphere flux standard deviation, %s'%qual_short
savedict['values']=deviations.std(axis=0)
savedict['dims']=dimdate+dimregs
savedict['comment']="This is the standard deviation on each parameter"
savedict['units']="mol region-1 s-1"
savedict['count']=next
ncf.AddData(savedict)
data = StateVector.EnsembleMembers[choicelag-1][0].ParameterValues*vectorocn # units of mole region-1 s-1 data = StateVector.EnsembleMembers[choicelag-1][0].ParameterValues*vectorocn # units of mole region-1 s-1
_saveInto(ncf, 'ocn_flux_%s'%qual_short, data, savedict=ncf.StandardVar(varname='ocn_flux_%s'%qual_short)
dims=dimdate+dimregs, count=next) savedict['values']=data
savedict['dims']=dimdate+dimregs
savedict['count']=next
ncf.AddData(savedict)
deviations = np.array([mem.ParameterValues*data for mem in members]) deviations = np.array([mem.ParameterValues*data for mem in members])
_saveInto(ncf, 'ocn_flux_%s_cov'%qual_short, deviations.tolist(), savedict=ncf.StandardVar(varname='ocn_flux_%s_cov'%qual_short)
dims=dimdate+dimmembers+dimregs, count=next, savedict['values']=deviations.tolist()
comment="This is the matrix square root, use (M x M^T)/(nmembers-1) to make covariance", savedict['dims']=dimdate+dimmembers+dimregs
units="mol region-1 s-1") savedict['comment']="This is the matrix square root, use (M x M^T)/(nmembers-1) to make covariance"
savedict['units']="mol region-1 s-1"
_saveInto(ncf, 'unknown', deviations.std(axis=0), savedict['count']=next
dims=dimdate+dimregs, count=next, ncf.AddData(savedict)
name = 'ocn_flux_%s_std'%qual_short,
long_name = 'Ocean flux standard deviation, %s'%qual_short, savedict=ncf.StandardVar('unknown')
comment = "This is the standard deviation on each parameter", savedict['name'] = 'ocn_flux_%s_std'%qual_short
units = "mol region-1 s-1") savedict['long_name'] = 'Ocean flux standard deviation, %s'%qual_short
savedict['values']=deviations.std(axis=0)
for name, data in [('fire_flux_imp', vectorfire), savedict['dims']=dimdate+dimregs
('fossil_flux_imp', vectorfossil)]: savedict['comment']="This is the standard deviation on each parameter"
_saveInto(ncf, name, data, savedict['units']="mol region-1 s-1"
dims=dimdate+dimregs, count=next) savedict['count']=next
ncf.AddData(savedict)
_saveInto(ncf, 'date', ncfdate,
dims=dimdate, count=next) data = vectorfire
savedict=ncf.StandardVar(varname='fire_flux_imp')
savedict['values']=data
savedict['dims']=dimdate+dimregs
savedict['count']=next
ncf.AddData(savedict)
data = vectorfossil
savedict=ncf.StandardVar(varname='fossil_flux_imp')
savedict['values']=data
savedict['dims']=dimdate+dimregs
savedict['count']=next
ncf.AddData(savedict)
savedict=ncf.StandardVar(varname='date')
savedict['values'] = ncfdate
savedict['dims'] = dimdate
savedict['count'] = next
ncf.AddData(savedict)
sys.stdout.write('.') sys.stdout.write('.')
sys.stdout.flush() sys.stdout.flush()
...@@ -456,8 +500,11 @@ def SaveWeeklyAvgTCData(DaCycle, StateVector): ...@@ -456,8 +500,11 @@ def SaveWeeklyAvgTCData(DaCycle, StateVector):
# First add the date for this cycle to the file, this grows the unlimited dimension # First add the date for this cycle to the file, this grows the unlimited dimension
_saveInto(ncf, 'date', ncfdate, savedict = ncf.StandardVar(varname='date')
dims=dimdate, count=index) savedict['values'] = ncfdate
savedict['dims'] = dimdate
savedict['count'] = index
dummy = ncf.AddData(savedict)
# Now convert other variables that were inside the flux_1x1 file # Now convert other variables that were inside the flux_1x1 file
...@@ -466,24 +513,31 @@ def SaveWeeklyAvgTCData(DaCycle, StateVector): ...@@ -466,24 +513,31 @@ def SaveWeeklyAvgTCData(DaCycle, StateVector):
data = ncf_in.GetVariable(vname)[index] data = ncf_in.GetVariable(vname)[index]
savedict = ncf.StandardVar(varname=vname)
if vname=='latitude': continue if vname=='latitude': continue
elif vname=='longitude': continue elif vname=='longitude': continue
elif vname=='date': continue elif vname=='date': continue
elif vname=='idate': continue elif vname=='idate': continue
elif 'cov' in vname: elif 'cov' in vname:
#AB? Only works with recent version of numpy, not with 1.4.1 installed on the suns #AB? Only works with recent version of numpy, not with 1.4.1 installed on the suns
#cov = data.transpose().dot(data) #cov = data.transpose().dot(data)
cov = np.dot(data.transpose(),data) cov = np.dot(data.transpose(),data)
tcdata = StateVector.VectorToTC(vectordata=cov,cov=True) # vector to TC tcdata = StateVector.VectorToTC(vectordata=cov,cov=True) # vector to TC
units = '[mol/region/s]**2'
dims = dimdate+dimregs+dimregs savedict['units'] = '[mol/region/s]**2'
savedict['dims'] = dimdate+dimregs+dimregs
else: else:
tcdata = StateVector.VectorToTC(vectordata=data) # vector to TC tcdata = StateVector.VectorToTC(vectordata=data) # vector to TC
dims = dimdate+dimregs
units = 'mol/region/s'
_saveInto(ncf, vname, tcdata, savedict['dims'] = dimdate+dimregs
dims=dims, units=units) savedict['units'] = 'mol/region/s'
savedict['values'] = tcdata
ncf.AddData(savedict)
ncf_in.close() ncf_in.close()
ncf.close() ncf.close()
...@@ -545,10 +599,12 @@ def SaveTCDataExt(rundat): ...@@ -545,10 +599,12 @@ def SaveTCDataExt(rundat):
dims=dimdate+dimregs dims=dimdate+dimregs
print vname,data.shape print vname,data.shape
_saveInto(ncf, vname, data.tolist(), savedict = ncf.StandardVar(varname=vname)
dims=dims, count=0, savedict['values'] = data.tolist()
units='mol/region/s', savedict['dims'] = dims
nsets=data.shape[0]) savedict['units'] = 'mol/region/s'
savedict['count'] = 0
ncf.AddData(savedict,nsets=data.shape[0])
ncf.close() ncf.close()
...@@ -601,10 +657,11 @@ def SaveEcoDataExt(rundat): ...@@ -601,10 +657,11 @@ def SaveEcoDataExt(rundat):
else: else:
print 'Dataset with unknown dimensions encountered in file: %s'%vname print 'Dataset with unknown dimensions encountered in file: %s'%vname
_saveInto(ncf, vname, data.tolist(), savedict=ncf.StandardVar(varname=vname)
dims=dims, savedict['units']=atts['units']
units=atts['units'], savedict['values']=data.tolist()
nsets=data.shape[0]) savedict['dims']=dims
ncf.AddData(savedict,nsets=data.shape[0])
ncf.close() ncf.close()
...@@ -695,16 +752,18 @@ def SaveTimeAvgData(rundat,infile,avg='monthly'): ...@@ -695,16 +752,18 @@ def SaveTimeAvgData(rundat,infile,avg='monthly'):
if ncf.StandardVar(sds)['name'] == 'unknown': if ncf.StandardVar(sds)['name'] == 'unknown':
invar = file.variables[sds] savedict = ncf.StandardVar(sds)
_saveInto(ncf, sds, data.tolist(), savedict['name'] = sds
dims=dims, count=0, savedict['values'] = data.tolist()
name=sds, savedict['dims'] = dims
units=invar.units, savedict['units'] = file.variables[sds].units
long_name=invar.long_name, savedict['long_name'] = file.variables[sds].long_name
comment=invar.comment, savedict['comment'] = file.variables[sds].comment
standard_name=invar.standard_name) savedict['standard_name'] = file.variables[sds].standard_name
sys.stdout.write('.') savedict['count'] =0
sys.stdout.flush() ncf.AddData(savedict)
sys.stdout.write('.')
sys.stdout.flush()
else: else:
...@@ -728,18 +787,22 @@ def SaveTimeAvgData(rundat,infile,avg='monthly'): ...@@ -728,18 +787,22 @@ def SaveTimeAvgData(rundat,infile,avg='monthly'):
count=count+1 count=count+1
if not ncf.has_date(date2num(dd)-dectime0): if not ncf.has_date(date2num(dd)-dectime0):
_saveInto(ncf, 'date', date2num(dd)-dectime0,
dims=dimdate, count=count)
units=file.variables[sds].units savedict=ncf.StandardVar('date')
savedict['values']=date2num(dd)-dectime0
savedict['dims']=dimdate
savedict['count']=count
ncf.AddData(savedict)
savedict=ncf.StandardVar(sds)
savedict['values']=data.tolist()
savedict['units']=file.variables[sds].units
if 'cov' in sds: if 'cov' in sds:
dimstmp=dimdate+dims+dims savedict['dims']=dimdate+dims+dims
else: else:
dimstmp=dimdate+dims savedict['dims']=dimdate+dims
savedict['count']=count
_saveInto(ncf, sds, data.tolist(), ncf.AddData(savedict)
dims=dimstmp, count=count,
units=units)
sys.stdout.write('.') sys.stdout.write('.')
sys.stdout.flush() sys.stdout.flush()
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment