Skip to content

GitLab

Commit dd098b72 authored by Peters, Wouter's avatar Peters, Wouter
Browse files

modules needed for IO with niceer CarbonTracker NetCDF interface

parent 300eef60
Show whitespace changes
Inline Side-by-side
ct_netcdf.py 0 → 100755
View file @ dd098b72
#!/usr/bin/env python
# ct_netcdf.py
"""
Author : peters
Revision History:
File created on 15 Oct 2008.
"""
import mysettings
import pycdf as CDF
import datetime as dt
import os
from numpy import array
if 'CTE' in mysettings.ct_version:
from ct_europe_info import *
else:
from ct_northamerica_info import *
std_savedict={'name':'unknown','values':[],'dims':(0,0,),'units':'','long_name':'','_FillValue':float(-999.),'comment':''}
class CT_CDF(CDF.CDF):
""" function opens a NetCDF/HDF/GRIB file for writing of output"""
def __init__(self,filename, method='read'):
if method not in ['read','write','create']:
raise ValueError, 'Method %s is not defined for a CarbonTracker NetCDF file object' % method
if method == 'read':
print 'Reading from file'
super(CDF.CDF,self).__init__(filename, CDF.NC.NOWRITE)
elif method == 'write':
#print 'Adding to existing file'
super(CT_CDF,self).__init__(filename, CDF.NC.WRITE|CDF.NC.CREATE)
self.AddCTHeader()
elif method == 'create':
#print 'Creating new file'
super(CT_CDF,self).__init__(filename, CDF.NC.WRITE|CDF.NC.TRUNC|CDF.NC.CREATE)
self.AddCTHeader()
def AddCTHeader(self):
self.automode()
#
setattr(self,'Institution',institution)
setattr(self,'Contact',email)
setattr(self,'URL',url)
setattr(self,'Source',source)
setattr(self,'Convention',conventions)
setattr(self,'Disclaimer',disclaimer)
setattr(self,'History',historytext)
def AddParamsDim(self,nparams):
self.automode()
dimparams=self.def_dim('nparameters',nparams)
return (dimparams,)
def AddMembersDim(self,nmembers):
self.automode()
dimmembers=self.def_dim('nmembers',nmembers)
return (dimmembers,)
def AddLagDim(self,nlag,unlimited=True):
self.automode()
if unlimited:
dimlag =self.def_dim('nlag',CDF.NC.UNLIMITED)
else:
dimlag=self.def_dim('nlag',nlag)
return (dimlag,)
def AddObsDim(self,nobs):
self.automode()
dimobs=self.def_dim('nobs',nobs)
return (dimobs,)
def AddLatLonDim(self,istart=0,iend=360,jstart=0,jend=180):
from numpy import arange, float64
if 'latitude' in self.dimensions(): return (self.dim('latitude'),self.dim('longitude'),) # already exists
lons=-180+arange(360)*1.0+0.5
lats=-90+arange(180)*1.0+0.5
#
lats=lats[jstart:jend]
lons=lons[istart:iend]
#
self.automode()
dimlon=self.def_dim('longitude',lons.shape[0])
dimlat=self.def_dim('latitude',lats.shape[0])
savedict=self.StandardVar(varname='latitude')
savedict['values']=lats.tolist()
savedict['actual_range']=(float(lats[0]),float(lats[-1]))
savedict['dims']=(dimlat,)
self.AddData(savedict)
savedict=self.StandardVar(varname='longitude')
savedict['values']=lons.tolist()
savedict['actual_range']=(float(lons[0]),float(lons[-1]))
savedict['dims']=(dimlon,)
self.AddData(savedict)
return (dimlat,dimlon,)
def AddRegionDim(self,rundat,type='eco'):
from tctools import olsonlabs, transnams, ext_transnams, ext_transcomps
from ecotools import ext_econams, ext_ecocomps
if type not in ['eco','eco_ext','tc','tc_ext','olson']: raise ValueError,'Type of dimension for regions requested (%s) is not possible' %type
dimname='regions_%s' % type
if dimname in self.dimensions(): return (self.dim(dimname),) # already exists
self.automode()
if type == 'olson':
dim = (self.def_dim(dimname,len(olsonlabs)),)
elif type == 'tc':
dim = (self.def_dim(dimname,len(transnams)),)
elif type == 'tc_ext':
dim = (self.def_dim(dimname,len(ext_transnams)),)
elif type == 'eco':
dim = (self.def_dim(dimname,rundat.ntotal_in_state),)
elif type == 'eco_ext':
dim = (self.def_dim(dimname,11*len(ext_econams)), )
# Automatically add region names as attributes
dummy = self.AddRegionDefs(rundat,type)
return dim
def AddDateDim(self):
self.automode()
if 'date' in self.dimensions(): return (self.dim('date'),)
return (self.def_dim('date',CDF.NC.UNLIMITED),)
def AddDateDimFormat(self):
self.automode()
if 'yyyymmddhhmmss' in self.dimensions(): return (self.dim('yyyymmddhhmmss'),) # already exists
return (self.def_dim('yyyymmddhhmmss',6),)
def AddRegionDefs(self,rundat,type='tc'):
""" add region definitions to NC file as global attributes """
from tctools import olsonnams, olsonlabs, transnams, ext_transnams, ext_transshort, ext_transcomps, LookUpName
from ecotools import ext_econams, ext_ecocomps
from oif import oifnams
import pyhdf.SD as HDF
n_land=rundat.n_land
n_ocean=rundat.n_ocean
if type not in ['tc','tc_ext','eco','eco_ext','olson']:
raise IOError,'Type (%s) not implemented in AddRegionDefs' % type
if type == 'olson':
# make attribute with olson ecoregion labels
for i,name in enumerate(olsonnams):
att = self.attr('OlsonEcosystem_%03d'%(i+1,))
att.put(CDF.NC.CHAR,name)
elif type == 'tc':
# make attribute with transcom region labels
for i,name in enumerate(transnams):
att = self.attr('TransComRegion_%03d'%(i+1,))
att.put(CDF.NC.CHAR,name)
elif type == 'tc_ext':
# get attributes from Andy's file and save to new file
for i,name in enumerate(ext_transnams):
lab='Aggregate_Region_%03d'%(i+1,)
setattr(self,lab,name)
for i,name in enumerate(ext_transcomps):
lab='Aggregate_Components_%03d'%(i+1,)
setattr(self,lab,name)
#
elif type == 'eco':
# make attribute with full eco region labels
for i in range(n_land):
att = self.attr('CarbonTrackerRegion_%03d'%(i+1,))
if i >= 209: i=i-19
name=transnams[i/19]+' : '+olsonlabs[i%19]
att.put(CDF.NC.CHAR,name)
for i in range(n_land,n_land+n_ocean):
att = self.attr('CarbonTrackerRegion_%03d'%(i+1,))
name=oifnams[i-n_land]
att.put(CDF.NC.CHAR,name)
att = self.attr('CarbonTrackerRegion_%03d'%(rundat.ntotal_in_state,))
name='Ice, Polar Desert, Unoptimized'
att.put(CDF.NC.CHAR,name)
elif type == 'eco_ext':
# make attribute with full extended eco region labels
for i,tcname in enumerate(transnams[0:11]):
for j in range(len(ext_econams)):
att = self.attr('Aggregate_Region_%02d'%(i*6+j+1,))
name=tcname+' : '+ext_econams[j]
att.put(CDF.NC.CHAR,name)
for i,tcname in enumerate(transnams[0:11]):
for j in range(len(ext_econams)):
lab = 'Aggregate_Components_%02d'%(i*6+j+1,)
setattr(self,lab,ext_ecocomps[j])
return None
def has_date(self,dd):
if self.inq_unlimlen() > 0:
if dd in self.GetVariable('date').tolist():
return True
else:
return False
else:
return False
def GetVariable(self,varname):
""" get variable from ncf file"""
return array(self.var(varname).get())
def StandardVar(self,varname):
""" return properties of standard variables """
import mysettings
if varname in mysettings.standard_variables.keys():
return mysettings.standard_variables[varname]
else:
return mysettings.standard_variables['unknown']
def AddData(self,datadict,nsets=1,silent=True):
""" add fields to file, at end of unlimited dimension"""
existing_vars=self.variables()
try:
next = datadict['count']
except:
next=0
if existing_vars.has_key(datadict['name']):
var = self.var(datadict['name'])
var[next:next+nsets]=datadict['values']
else:
if not silent: print 'Creating new dataset: '+datadict['name']
if datadict.has_key('dtype'):
if datadict['dtype'] == 'int':
var = self.def_var(datadict['name'],CDF.NC.INT,datadict['dims'])
elif datadict['dtype'] == 'char':
var = self.def_var(datadict['name'],CDF.NC.CHAR,datadict['dims'])
elif datadict['dtype'] == 'double':
var = self.def_var(datadict['name'],CDF.NC.DOUBLE,datadict['dims'])
else:
var = self.def_var(datadict['name'],CDF.NC.FLOAT,datadict['dims'])
else:
var = self.def_var(datadict['name'],CDF.NC.FLOAT,datadict['dims'])
for k,v in datadict.iteritems():
if k not in ['name','dims','values','_FillValue','count']:
setattr(var,k,v)
if var.isrecord():
var[next:next+nsets]=datadict['values']
else:
var[:]=datadict['values']
def GetVariable(file,varname):
""" get variable from HDF file"""
return array(file.select(varname).get())
def CreateDirs(rundat,dirname):
dirname=os.path.join(rundat.outputdir,dirname)
if not os.path.exists(dirname):
print "Creating new output directory "+dirname
os.makedirs(dirname)
else:
print 'Writing files to directory: %s'%(dirname,)
return dirname
if __name__ == '__main__':
try:
os.remove('test.nc')
except:
pass
ncf=CT_CDF('test.nc','create')
dimgrid=ncf.AddLatLonDim()
dimdate=ncf.AddDateDim()
dimidate=ncf.AddDateDimFormat()
mysettings.py 0 → 100755
View file @ dd098b72
from pylab import *
import os
import rc
from matplotlib.font_manager import FontProperties
homedir=os.environ["HOME"]
username=os.environ["USER"]
boilerplatedir='/ct/tools/shared/boilerplate'
rcinfo=rc.read(os.path.join(homedir,'.carbontrackerrc'))
ct_inputdir=''
ct_outputdir=''
ct_pylibdir=''
ct_version=''
ct_dir_scheme='wouter'
unicodefontfilename=None
if rcinfo.has_key('carbontracker.std.inputdir'): ct_inputdir=rcinfo['carbontracker.std.inputdir']
if rcinfo.has_key('carbontracker.std.outputdir'): ct_outputdir=rcinfo['carbontracker.std.outputdir']
if rcinfo.has_key('ct.shared.dir'): ctshareddir=rcinfo['ct.shared.dir']
if rcinfo.has_key('python.std.libdir'): pylibdir=rcinfo['python.std.libdir']
if rcinfo.has_key('unicodefont'): unicodefontfilename=rcinfo['unicodefont']
if rcinfo.has_key('carbontracker.version'): ct_version=rcinfo['carbontracker.version']
if rcinfo.has_key('carbontracker.dir.scheme'): ct_dir_scheme=rcinfo['carbontracker.dir.scheme']
if not os.path.exists(boilerplatedir):
if rcinfo.has_key('boilerplate.dir'): boilerplatedir=rcinfo['boilerplate.dir']
unicodefont=FontProperties(fname=unicodefontfilename)
standard_variables = { 'bio_flux_prior' : {'name' : 'bio_flux_prior',\
'units' : 'mol m-2 s-1' ,\
'long_name' : 'Surface flux of carbon dioxide, terrestrial vegetation, not optimized ', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : 'surface_carbon_dioxide_mole_flux', \
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'bio_flux_opt' : {'name' : 'bio_flux_opt',\
'units' : 'mol m-2 s-1' ,\
'long_name' : 'Surface flux of carbon dioxide, terrestrial biosphere , optimized ', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : 'surface_carbon_dioxide_mole_flux', \
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'ocn_flux_prior' : {'name' : 'ocn_flux_prior',\
'units' : 'mol m-2 s-1' ,\
'long_name' : 'Surface flux of carbon dioxide, open ocean , not optimized ', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : 'surface_carbon_dioxide_mole_flux', \
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'ocn_flux_opt' : {'name' : 'ocn_flux_opt',\
'units' : 'mol m-2 s-1' ,\
'long_name' : 'Surface flux of carbon dioxide, open ocean , optimized ', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : 'surface_carbon_dioxide_mole_flux', \
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'fossil_flux_imp' : {'name' : 'fossil_flux_imp',\
'units' : 'mol m-2 s-1' ,\
'long_name' : 'Surface flux of carbon dioxide, fossil fuel burning , imposed ', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : 'surface_carbon_dioxide_mole_flux', \
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'fire_flux_imp' : {'name' : 'fire_flux_imp',\
'units' : 'mol m-2 s-1' ,\
'long_name' : 'Surface flux of carbon dioxide, biomass burning , imposed ', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : 'surface_carbon_dioxide_mole_flux', \
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'bio_flux_prior_cov' : {'name' : 'bio_flux_prior_cov',\
'units' : 'mol2 region-2 s-2' ,\
'long_name' : 'Covariance of surface flux of carbon dioxide, terrestrial vegetation , not optimized ', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : '', \
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'bio_flux_opt_cov' : {'name' : 'bio_flux_opt_cov',\
'units' : 'mol2 region-2 s-2' ,\
'long_name' : 'Covariance of surface flux of carbon dioxide, terrestrial vegetation , optimized ', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : '', \
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'ocn_flux_prior_cov' : {'name' : 'ocn_flux_prior_cov',\
'units' : 'mol2 region-2 s-2' ,\
'long_name' : 'Covariance of surface flux of carbon dioxide, open ocean , not optimized ', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : '', \
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'ocn_flux_opt_cov' : {'name' : 'ocn_flux_opt_cov',\
'units' : 'mol2 region-2 s-2' ,\
'long_name' : 'Covariance of surface flux of carbon dioxide, open ocean , optimized ', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : '', \
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'decimal_date' : {'name' : 'decimal_date',\
'units' : 'years' ,\
'long_name' : 'dates and times', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : 'date', \
'dims' : (), \
'dtype' : 'double', \
'values' : [], \
'count' : 0 \
} , \
'date' : {'name' : 'date',\
'units' : 'days since 2000-01-01 00:00:00 UTC' ,\
'long_name' : 'UTC dates and times', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : 'date', \
'dims' : (), \
'dtype' : 'double', \
'values' : [], \
'count' : 0 \
} , \
'idate' : {'name' : 'idate',\
'units' : 'yyyy MM dd hh mm ss ' ,\
'long_name' : 'integer components of date and time', \
'standard_name' : 'calendar_components', \
'comment' : 'time-interval average, centered on times in the date axis', \
'dims' : (), \
'dtype' : 'int', \
'values' : [], \
'count' : 0 \
} , \
'latitude' : {'name' : 'latitude',\
'units' : 'degrees_north ' ,\
'long_name' : 'latitude', \
'standard_name' : 'latitude', \
'comment' : 'center of interval',\
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'longitude' : {'name' : 'longitude',\
'units' : 'degrees_east ' ,\
'long_name' : 'longitude', \
'standard_name' : 'longitude', \
'comment' : 'center of interval',\
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'height' : {'name' : 'height',\
'units' : 'masl ' ,\
'long_name' : 'height_above_ground_level', \
'standard_name' : 'height_above_ground_level', \
'comment' : 'value is meters above sea level',\
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'co2' : {'name' : 'co2',\
'units' : 'micromol mol-1 ' ,\
'long_name' : 'mole_fraction_of_carbon_dioxide_in_air', \
'standard_name' : 'mole_fraction_of_carbon_dioxide_in_air', \
'comment' : '',\
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'meanstate' : {'name' : 'statevectormean',\
'units' : 'unitless' ,\
'long_name' : 'mean_value_of_state_vector', \
'standard_name' : 'mean_value_of_state_vector', \
'comment' : '',\
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'ensemblestate': {'name' : 'statevectorensemble',\
'units' : 'unitless' ,\
'long_name' : 'ensemble_value_of_state_vector', \
'standard_name' : 'ensemble_value_of_state_vector', \
'comment' : '',\
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'unknown' : {'name' : '',\
'units' : '' ,\
'long_name' : '', \
'standard_name' : '', \
'comment' : '',\
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment