Commit dd098b72 authored by Peters, Wouter's avatar Peters, Wouter
Browse files

modules needed for IO with niceer CarbonTracker NetCDF interface

parent 300eef60
#!/usr/bin/env python
# ct_netcdf.py
"""
Author : peters
Revision History:
File created on 15 Oct 2008.
"""
import mysettings
import pycdf as CDF
import datetime as dt
import os
from numpy import array
if 'CTE' in mysettings.ct_version:
from ct_europe_info import *
else:
from ct_northamerica_info import *
std_savedict={'name':'unknown','values':[],'dims':(0,0,),'units':'','long_name':'','_FillValue':float(-999.),'comment':''}
class CT_CDF(CDF.CDF):
""" function opens a NetCDF/HDF/GRIB file for writing of output"""
def __init__(self,filename, method='read'):
if method not in ['read','write','create']:
raise ValueError, 'Method %s is not defined for a CarbonTracker NetCDF file object' % method
if method == 'read':
print 'Reading from file'
super(CDF.CDF,self).__init__(filename, CDF.NC.NOWRITE)
elif method == 'write':
#print 'Adding to existing file'
super(CT_CDF,self).__init__(filename, CDF.NC.WRITE|CDF.NC.CREATE)
self.AddCTHeader()
elif method == 'create':
#print 'Creating new file'
super(CT_CDF,self).__init__(filename, CDF.NC.WRITE|CDF.NC.TRUNC|CDF.NC.CREATE)
self.AddCTHeader()
def AddCTHeader(self):
self.automode()
#
setattr(self,'Institution',institution)
setattr(self,'Contact',email)
setattr(self,'URL',url)
setattr(self,'Source',source)
setattr(self,'Convention',conventions)
setattr(self,'Disclaimer',disclaimer)
setattr(self,'History',historytext)
def AddParamsDim(self,nparams):
self.automode()
dimparams=self.def_dim('nparameters',nparams)
return (dimparams,)
def AddMembersDim(self,nmembers):
self.automode()
dimmembers=self.def_dim('nmembers',nmembers)
return (dimmembers,)
def AddLagDim(self,nlag,unlimited=True):
self.automode()
if unlimited:
dimlag =self.def_dim('nlag',CDF.NC.UNLIMITED)
else:
dimlag=self.def_dim('nlag',nlag)
return (dimlag,)
def AddObsDim(self,nobs):
self.automode()
dimobs=self.def_dim('nobs',nobs)
return (dimobs,)
def AddLatLonDim(self,istart=0,iend=360,jstart=0,jend=180):
from numpy import arange, float64
if 'latitude' in self.dimensions(): return (self.dim('latitude'),self.dim('longitude'),) # already exists
lons=-180+arange(360)*1.0+0.5
lats=-90+arange(180)*1.0+0.5
#
lats=lats[jstart:jend]
lons=lons[istart:iend]
#
self.automode()
dimlon=self.def_dim('longitude',lons.shape[0])
dimlat=self.def_dim('latitude',lats.shape[0])
savedict=self.StandardVar(varname='latitude')
savedict['values']=lats.tolist()
savedict['actual_range']=(float(lats[0]),float(lats[-1]))
savedict['dims']=(dimlat,)
self.AddData(savedict)
savedict=self.StandardVar(varname='longitude')
savedict['values']=lons.tolist()
savedict['actual_range']=(float(lons[0]),float(lons[-1]))
savedict['dims']=(dimlon,)
self.AddData(savedict)
return (dimlat,dimlon,)
def AddRegionDim(self,rundat,type='eco'):
from tctools import olsonlabs, transnams, ext_transnams, ext_transcomps
from ecotools import ext_econams, ext_ecocomps
if type not in ['eco','eco_ext','tc','tc_ext','olson']: raise ValueError,'Type of dimension for regions requested (%s) is not possible' %type
dimname='regions_%s' % type
if dimname in self.dimensions(): return (self.dim(dimname),) # already exists
self.automode()
if type == 'olson':
dim = (self.def_dim(dimname,len(olsonlabs)),)
elif type == 'tc':
dim = (self.def_dim(dimname,len(transnams)),)
elif type == 'tc_ext':
dim = (self.def_dim(dimname,len(ext_transnams)),)
elif type == 'eco':
dim = (self.def_dim(dimname,rundat.ntotal_in_state),)
elif type == 'eco_ext':
dim = (self.def_dim(dimname,11*len(ext_econams)), )
# Automatically add region names as attributes
dummy = self.AddRegionDefs(rundat,type)
return dim
def AddDateDim(self):
self.automode()
if 'date' in self.dimensions(): return (self.dim('date'),)
return (self.def_dim('date',CDF.NC.UNLIMITED),)
def AddDateDimFormat(self):
self.automode()
if 'yyyymmddhhmmss' in self.dimensions(): return (self.dim('yyyymmddhhmmss'),) # already exists
return (self.def_dim('yyyymmddhhmmss',6),)
def AddRegionDefs(self,rundat,type='tc'):
""" add region definitions to NC file as global attributes """
from tctools import olsonnams, olsonlabs, transnams, ext_transnams, ext_transshort, ext_transcomps, LookUpName
from ecotools import ext_econams, ext_ecocomps
from oif import oifnams
import pyhdf.SD as HDF
n_land=rundat.n_land
n_ocean=rundat.n_ocean
if type not in ['tc','tc_ext','eco','eco_ext','olson']:
raise IOError,'Type (%s) not implemented in AddRegionDefs' % type
if type == 'olson':
# make attribute with olson ecoregion labels
for i,name in enumerate(olsonnams):
att = self.attr('OlsonEcosystem_%03d'%(i+1,))
att.put(CDF.NC.CHAR,name)
elif type == 'tc':
# make attribute with transcom region labels
for i,name in enumerate(transnams):
att = self.attr('TransComRegion_%03d'%(i+1,))
att.put(CDF.NC.CHAR,name)
elif type == 'tc_ext':
# get attributes from Andy's file and save to new file
for i,name in enumerate(ext_transnams):
lab='Aggregate_Region_%03d'%(i+1,)
setattr(self,lab,name)
for i,name in enumerate(ext_transcomps):
lab='Aggregate_Components_%03d'%(i+1,)
setattr(self,lab,name)
#
elif type == 'eco':
# make attribute with full eco region labels
for i in range(n_land):
att = self.attr('CarbonTrackerRegion_%03d'%(i+1,))
if i >= 209: i=i-19
name=transnams[i/19]+' : '+olsonlabs[i%19]
att.put(CDF.NC.CHAR,name)
for i in range(n_land,n_land+n_ocean):
att = self.attr('CarbonTrackerRegion_%03d'%(i+1,))
name=oifnams[i-n_land]
att.put(CDF.NC.CHAR,name)
att = self.attr('CarbonTrackerRegion_%03d'%(rundat.ntotal_in_state,))
name='Ice, Polar Desert, Unoptimized'
att.put(CDF.NC.CHAR,name)
elif type == 'eco_ext':
# make attribute with full extended eco region labels
for i,tcname in enumerate(transnams[0:11]):
for j in range(len(ext_econams)):
att = self.attr('Aggregate_Region_%02d'%(i*6+j+1,))
name=tcname+' : '+ext_econams[j]
att.put(CDF.NC.CHAR,name)
for i,tcname in enumerate(transnams[0:11]):
for j in range(len(ext_econams)):
lab = 'Aggregate_Components_%02d'%(i*6+j+1,)
setattr(self,lab,ext_ecocomps[j])
return None
def has_date(self,dd):
if self.inq_unlimlen() > 0:
if dd in self.GetVariable('date').tolist():
return True
else:
return False
else:
return False
def GetVariable(self,varname):
""" get variable from ncf file"""
return array(self.var(varname).get())
def StandardVar(self,varname):
""" return properties of standard variables """
import mysettings
if varname in mysettings.standard_variables.keys():
return mysettings.standard_variables[varname]
else:
return mysettings.standard_variables['unknown']
def AddData(self,datadict,nsets=1,silent=True):
""" add fields to file, at end of unlimited dimension"""
existing_vars=self.variables()
try:
next = datadict['count']
except:
next=0
if existing_vars.has_key(datadict['name']):
var = self.var(datadict['name'])
var[next:next+nsets]=datadict['values']
else:
if not silent: print 'Creating new dataset: '+datadict['name']
if datadict.has_key('dtype'):
if datadict['dtype'] == 'int':
var = self.def_var(datadict['name'],CDF.NC.INT,datadict['dims'])
elif datadict['dtype'] == 'char':
var = self.def_var(datadict['name'],CDF.NC.CHAR,datadict['dims'])
elif datadict['dtype'] == 'double':
var = self.def_var(datadict['name'],CDF.NC.DOUBLE,datadict['dims'])
else:
var = self.def_var(datadict['name'],CDF.NC.FLOAT,datadict['dims'])
else:
var = self.def_var(datadict['name'],CDF.NC.FLOAT,datadict['dims'])
for k,v in datadict.iteritems():
if k not in ['name','dims','values','_FillValue','count']:
setattr(var,k,v)
if var.isrecord():
var[next:next+nsets]=datadict['values']
else:
var[:]=datadict['values']
def GetVariable(file,varname):
""" get variable from HDF file"""
return array(file.select(varname).get())
def CreateDirs(rundat,dirname):
dirname=os.path.join(rundat.outputdir,dirname)
if not os.path.exists(dirname):
print "Creating new output directory "+dirname
os.makedirs(dirname)
else:
print 'Writing files to directory: %s'%(dirname,)
return dirname
if __name__ == '__main__':
try:
os.remove('test.nc')
except:
pass
ncf=CT_CDF('test.nc','create')
dimgrid=ncf.AddLatLonDim()
dimdate=ncf.AddDateDim()
dimidate=ncf.AddDateDimFormat()
from pylab import *
import os
import rc
from matplotlib.font_manager import FontProperties
homedir=os.environ["HOME"]
username=os.environ["USER"]
boilerplatedir='/ct/tools/shared/boilerplate'
rcinfo=rc.read(os.path.join(homedir,'.carbontrackerrc'))
ct_inputdir=''
ct_outputdir=''
ct_pylibdir=''
ct_version=''
ct_dir_scheme='wouter'
unicodefontfilename=None
if rcinfo.has_key('carbontracker.std.inputdir'): ct_inputdir=rcinfo['carbontracker.std.inputdir']
if rcinfo.has_key('carbontracker.std.outputdir'): ct_outputdir=rcinfo['carbontracker.std.outputdir']
if rcinfo.has_key('ct.shared.dir'): ctshareddir=rcinfo['ct.shared.dir']
if rcinfo.has_key('python.std.libdir'): pylibdir=rcinfo['python.std.libdir']
if rcinfo.has_key('unicodefont'): unicodefontfilename=rcinfo['unicodefont']
if rcinfo.has_key('carbontracker.version'): ct_version=rcinfo['carbontracker.version']
if rcinfo.has_key('carbontracker.dir.scheme'): ct_dir_scheme=rcinfo['carbontracker.dir.scheme']
if not os.path.exists(boilerplatedir):
if rcinfo.has_key('boilerplate.dir'): boilerplatedir=rcinfo['boilerplate.dir']
unicodefont=FontProperties(fname=unicodefontfilename)
standard_variables = { 'bio_flux_prior' : {'name' : 'bio_flux_prior',\
'units' : 'mol m-2 s-1' ,\
'long_name' : 'Surface flux of carbon dioxide, terrestrial vegetation, not optimized ', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : 'surface_carbon_dioxide_mole_flux', \
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'bio_flux_opt' : {'name' : 'bio_flux_opt',\
'units' : 'mol m-2 s-1' ,\
'long_name' : 'Surface flux of carbon dioxide, terrestrial biosphere , optimized ', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : 'surface_carbon_dioxide_mole_flux', \
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'ocn_flux_prior' : {'name' : 'ocn_flux_prior',\
'units' : 'mol m-2 s-1' ,\
'long_name' : 'Surface flux of carbon dioxide, open ocean , not optimized ', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : 'surface_carbon_dioxide_mole_flux', \
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'ocn_flux_opt' : {'name' : 'ocn_flux_opt',\
'units' : 'mol m-2 s-1' ,\
'long_name' : 'Surface flux of carbon dioxide, open ocean , optimized ', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : 'surface_carbon_dioxide_mole_flux', \
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'fossil_flux_imp' : {'name' : 'fossil_flux_imp',\
'units' : 'mol m-2 s-1' ,\
'long_name' : 'Surface flux of carbon dioxide, fossil fuel burning , imposed ', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : 'surface_carbon_dioxide_mole_flux', \
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'fire_flux_imp' : {'name' : 'fire_flux_imp',\
'units' : 'mol m-2 s-1' ,\
'long_name' : 'Surface flux of carbon dioxide, biomass burning , imposed ', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : 'surface_carbon_dioxide_mole_flux', \
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'bio_flux_prior_cov' : {'name' : 'bio_flux_prior_cov',\
'units' : 'mol2 region-2 s-2' ,\
'long_name' : 'Covariance of surface flux of carbon dioxide, terrestrial vegetation , not optimized ', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : '', \
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'bio_flux_opt_cov' : {'name' : 'bio_flux_opt_cov',\
'units' : 'mol2 region-2 s-2' ,\
'long_name' : 'Covariance of surface flux of carbon dioxide, terrestrial vegetation , optimized ', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : '', \
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'ocn_flux_prior_cov' : {'name' : 'ocn_flux_prior_cov',\
'units' : 'mol2 region-2 s-2' ,\
'long_name' : 'Covariance of surface flux of carbon dioxide, open ocean , not optimized ', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : '', \
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'ocn_flux_opt_cov' : {'name' : 'ocn_flux_opt_cov',\
'units' : 'mol2 region-2 s-2' ,\
'long_name' : 'Covariance of surface flux of carbon dioxide, open ocean , optimized ', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : '', \
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'decimal_date' : {'name' : 'decimal_date',\
'units' : 'years' ,\
'long_name' : 'dates and times', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : 'date', \
'dims' : (), \
'dtype' : 'double', \
'values' : [], \
'count' : 0 \
} , \
'date' : {'name' : 'date',\
'units' : 'days since 2000-01-01 00:00:00 UTC' ,\
'long_name' : 'UTC dates and times', \
'comment' : 'time-interval average, centered on times in the date axis', \
'standard_name' : 'date', \
'dims' : (), \
'dtype' : 'double', \
'values' : [], \
'count' : 0 \
} , \
'idate' : {'name' : 'idate',\
'units' : 'yyyy MM dd hh mm ss ' ,\
'long_name' : 'integer components of date and time', \
'standard_name' : 'calendar_components', \
'comment' : 'time-interval average, centered on times in the date axis', \
'dims' : (), \
'dtype' : 'int', \
'values' : [], \
'count' : 0 \
} , \
'latitude' : {'name' : 'latitude',\
'units' : 'degrees_north ' ,\
'long_name' : 'latitude', \
'standard_name' : 'latitude', \
'comment' : 'center of interval',\
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'longitude' : {'name' : 'longitude',\
'units' : 'degrees_east ' ,\
'long_name' : 'longitude', \
'standard_name' : 'longitude', \
'comment' : 'center of interval',\
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'height' : {'name' : 'height',\
'units' : 'masl ' ,\
'long_name' : 'height_above_ground_level', \
'standard_name' : 'height_above_ground_level', \
'comment' : 'value is meters above sea level',\
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'co2' : {'name' : 'co2',\
'units' : 'micromol mol-1 ' ,\
'long_name' : 'mole_fraction_of_carbon_dioxide_in_air', \
'standard_name' : 'mole_fraction_of_carbon_dioxide_in_air', \
'comment' : '',\
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'meanstate' : {'name' : 'statevectormean',\
'units' : 'unitless' ,\
'long_name' : 'mean_value_of_state_vector', \
'standard_name' : 'mean_value_of_state_vector', \
'comment' : '',\
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'ensemblestate': {'name' : 'statevectorensemble',\
'units' : 'unitless' ,\
'long_name' : 'ensemble_value_of_state_vector', \
'standard_name' : 'ensemble_value_of_state_vector', \
'comment' : '',\
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
'unknown' : {'name' : '',\
'units' : '' ,\
'long_name' : '', \
'standard_name' : '', \
'comment' : '',\
'dims' : (), \
'values' : [], \
'count' : 0 \
} , \
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment