Commit 7e83a098 authored by brunner's avatar brunner
Browse files

No commit message

No commit message
parent 469624cd
......@@ -110,7 +110,7 @@ def write_mole_fractions(dacycle):
fc_simulated = ncf_fc_in.get_variable('modelsamplesmean_prior')
fc_simulated_ens = ncf_fc_in.get_variable('modelsamplesdeviations_prior')
fc_flag = ncf_fc_in.get_variable('flag')
if not dacycle.dasystem.has_key('opt.algorithm'):
if 'modeldatamismatchvariance' not in dacycle.dasystem:
fc_r = ncf_fc_in.get_variable('modeldatamismatchvariance')
fc_hphtr = ncf_fc_in.get_variable('totalmolefractionvariance')
elif dacycle.dasystem['opt.algorithm'] == 'serial':
......@@ -172,10 +172,10 @@ def write_mole_fractions(dacycle):
# get nobs dimension
if ncf_out.dimensions.has_key('id'):
if 'id' in ncf_out.dimensions:
dimidob = ncf_out.dimensions['id']
dimid = ('id',)
elif ncf_out.dimensions.has_key('obs'):
elif 'obs' in ncf_out.dimensions:
dimidob = ncf_out.dimensions['obs']
dimid = ('obs',)
......@@ -273,11 +273,11 @@ def write_mole_fractions(dacycle):
# Get existing file obs_nums to determine match to local obs_nums
if ncf_out.variables.has_key('merge_num'):
if 'merge_num' in ncf_out.variables:
file_obs_nums = ncf_out.get_variable('merge_num')
elif ncf_out.variables.has_key('obspack_num'):
elif 'obspack_num' in ncf_out.variables:
file_obs_nums = ncf_out.get_variable('obspack_num')
elif ncf_out.variables.has_key('id'):
elif 'id' in ncf_out.variables:
file_obs_nums = ncf_out.get_variable('id')
# Get all obs_nums related to this file, determine their indices in the local arrays
......
......@@ -70,7 +70,7 @@ class DaSystem(dict):
"""
This method loads a DA System Info rc-file with settings for this simulation
"""
for k, v in rc.read(rcfilename).iteritems():
for k, v in rc.read(rcfilename).items():
self[k] = v
logging.debug("DA System Info rc-file (%s) loaded successfully" % rcfilename)
......@@ -82,17 +82,17 @@ class DaSystem(dict):
"""
needed_rc_items = {}
for k, v in self.iteritems():
for k, v in self.items():
if v == 'True' :
self[k] = True
if v == 'False':
self[k] = False
for key in needed_rc_items:
if not self.has_key(key):
if key not in self:
msg = 'Missing a required value in rc-file : %s' % key
logging.error(msg)
raise IOError, msg
raise IOError(msg)
logging.debug('DA System Info settings have been validated succesfully')
################### End Class DaSystem ###################
......
......@@ -95,11 +95,11 @@ class Platform(object):
template += """#$ -hold_jid depends \n"""
# First replace from passed dictionary
for k, v in joboptions.iteritems():
for k, v in joboptions.items():
while k in template:
template = template.replace(k, v)
# Fill remaining values with std_options
for k, v in std_joboptions.iteritems():
for k, v in std_joboptions.items():
while k in template:
template = template.replace(k, v)
return template
......
......@@ -156,7 +156,7 @@ class StateVector(object):
# These list objects hold the data for each time step of lag in the system. Note that the ensembles for each time step consist
# of lists of EnsembleMember objects, we define member 0 as the mean of the distribution and n=1,...,nmembers as the spread.
self.ensemble_members = range(self.nlag)
self.ensemble_members = list(range(self.nlag))
for n in range(self.nlag):
self.ensemble_members[n] = []
......@@ -500,7 +500,7 @@ class StateVector(object):
raise ValueError
result = np.zeros((self.nparams,), float)
for k, v in self.griddict.iteritems():
for k, v in self.griddict.items():
#print k,k-1,result.shape, v
if method == "avg":
result[k - 1] = griddata.take(v).mean()
......@@ -528,7 +528,7 @@ class StateVector(object):
"""
result = np.zeros(self.gridmap.shape, float)
for k, v in self.griddict.iteritems():
for k, v in self.griddict.items():
#print k,v
result.put(v, vectordata[k - 1])
return result
......
......@@ -46,14 +46,14 @@ class CO2DaSystem(DaSystem):
'regtype']
for k, v in self.iteritems():
for k, v in self.items():
if v == 'True' :
self[k] = True
if v == 'False':
self[k] = False
for key in needed_rc_items:
if not self.has_key(key):
if key not in self:
logging.warning('Missing a required value in rc-file : %s' % key)
logging.debug('DA System Info settings have been validated succesfully')
......
......@@ -56,7 +56,7 @@ class CO2Observations(Observations):
if not os.path.exists(filename):
msg = 'Could not find the required observation input file (%s) ' % filename
logging.error(msg)
raise IOError, msg
raise IOError(msg)
else:
self.obs_filename = filename
self.datalist = []
......@@ -118,7 +118,7 @@ class CO2Observations(Observations):
logging.error(msg)
logging.error("Did the sampling step succeed?")
logging.error("...exiting")
raise IOError, msg
raise IOError(msg)
ncf = io.ct_read(filename, method='read')
ids = ncf.get_variable('obs_num')
......@@ -129,7 +129,7 @@ class CO2Observations(Observations):
obs_ids = self.getvalues('id')
obs_ids = obs_ids.tolist()
ids = map(int, ids)
ids = list(map(int, ids))
missing_samples = []
......@@ -265,7 +265,7 @@ class CO2Observations(Observations):
if not os.path.exists(filename):
msg = 'Could not find the required sites.rc input file (%s)' % filename
logging.error(msg)
raise IOError, msg
raise IOError(msg)
else:
self.sites_file = filename
......@@ -308,7 +308,7 @@ class CO2Observations(Observations):
for obs in self.datalist:
obs.mdm = 1000.0 # default is very high model-data-mismatch, until explicitly set by script
if site_info.has_key(obs.code):
if obs.code in site_info:
logging.debug("Observation found (%s)" % obs.code)
obs.mdm = site_info[obs.code]['error'] * self.global_R_scaling
obs.may_localize = site_info[obs.code]['may_localize']
......
......@@ -24,7 +24,7 @@ import os
import sys
import logging
import datetime as dtm
from string import strip
#from string import strip
from numpy import array, logical_and
sys.path.append(os.getcwd())
......@@ -55,7 +55,7 @@ class ObsPackObservations(Observations):
if not os.path.exists(op_dir):
msg = 'Could not find the required ObsPack distribution (%s) ' % op_dir
logging.error(msg)
raise IOError, msg
raise IOError(msg)
else:
self.obspack_dir = op_dir
self.obspack_id = op_id
......@@ -102,7 +102,7 @@ class ObsPackObservations(Observations):
ids = ncf.get_variable('obspack_num').take(subselect) # or should we propagate obs_num which is not unique across datasets??
evn = ncf.get_variable('obspack_id').take(subselect, axis=0)
evn = [s.tostring().lower() for s in evn]
evn = map(strip, evn)
evn = map(str, evn)
site = ncf.get_attribute('site_code')
lats = ncf.get_variable('latitude').take(subselect, axis=0)
lons = ncf.get_variable('longitude').take(subselect, axis=0)
......@@ -113,7 +113,7 @@ class ObsPackObservations(Observations):
strategy = 1
flags = ncf.get_variable('qc_flag').take(subselect, axis=0)
flags = [s.tostring().lower() for s in flags]
flags = map(strip, flags)
flags = map(str, flags)
flags = [int(f == '...') for f in flags]
ncf.close()
......@@ -132,7 +132,7 @@ class ObsPackObservations(Observations):
logging.error(msg)
logging.error("Did the sampling step succeed?")
logging.error("...exiting")
raise IOError, msg
raise IOError(msg)
ncf = io.ct_read(filename, method='read')
ids = ncf.get_variable('obs_num')
......@@ -141,7 +141,7 @@ class ObsPackObservations(Observations):
logging.info("Successfully read data from model sample file (%s)" % filename)
obs_ids = self.getvalues('id').tolist()
ids = map(int, ids)
ids = list(map(int, ids))
missing_samples = []
......@@ -296,7 +296,7 @@ class ObsPackObservations(Observations):
if not os.path.exists(filename):
msg = 'Could not find the required sites.rc input file (%s) ' % filename
logging.error(msg)
raise IOError, msg
raise IOError(msg)
else:
self.sites_file = filename
......@@ -348,7 +348,7 @@ class ObsPackObservations(Observations):
identifier = name_convert(name="%s_%s_%s" % (site.lower(), method.lower(), lab.lower(),), to='GV')
if site_info.has_key(identifier):
if identifier in site_info:
logging.debug("Observation found (%s, %s)" % (obs.code, identifier))
obs.mdm = site_info[identifier]['error'] * self.global_R_scaling
obs.may_localize = site_info[identifier]['may_localize']
......
......@@ -52,7 +52,7 @@ class ObsPackObservations(Observations):
if not os.path.exists(op_dir):
msg = 'Could not find the required ObsPack distribution (%s) ' % op_dir
logging.error(msg)
raise IOError, msg
raise IOError(msg)
else:
self.obspack_dir = op_dir
self.obspack_id = op_id
......@@ -101,7 +101,7 @@ class ObsPackObservations(Observations):
obspacknum = ncf.get_variable('obspack_num').take(subselect) # or should we propagate obs_num which is not unique across datasets??
obspackid = ncf.get_variable('obspack_id').take(subselect, axis=0)
obspackid = [s.tostring().lower() for s in obspackid]
obspackid = map(strip, obspackid)
obspackid = list(map(str.strip,str(obspackid)))
datasetname = ncfile # use full name of dataset to propagate for clarity
lats = ncf.get_variable('latitude').take(subselect, axis=0)
lons = ncf.get_variable('longitude').take(subselect, axis=0)
......@@ -127,7 +127,7 @@ class ObsPackObservations(Observations):
logging.error(msg)
logging.error("Did the sampling step succeed?")
logging.error("...exiting")
raise IOError, msg
raise IOError(msg)
ncf = io.ct_read(filename, method='read')
ids = ncf.get_variable('obs_num')
......@@ -136,7 +136,7 @@ class ObsPackObservations(Observations):
logging.info("Successfully read data from model sample file (%s)" % filename)
obs_ids = self.getvalues('id').tolist()
ids = map(int, ids)
ids = list(map(int, ids))
missing_samples = []
......@@ -173,7 +173,7 @@ class ObsPackObservations(Observations):
dim10char = f.add_dim('string_of10chars', 10)
dimcalcomp = f.add_dim('calendar_components', 6)
for key, value in self.site_move.iteritems():
for key, value in self.site_move.items():
msg = "Site is moved by %3.2f degrees latitude and %3.2f degrees longitude" % value
f.add_attribute(key, msg)
......@@ -296,7 +296,7 @@ class ObsPackObservations(Observations):
if not os.path.exists(filename):
msg = 'Could not find the required sites.rc input file (%s) ' % filename
logging.error(msg)
raise IOError, msg
raise IOError(msg)
else:
self.sites_file = filename
......@@ -325,7 +325,7 @@ class ObsPackObservations(Observations):
site_move = {}
site_hourly = {} # option added to include only certain hours of the day (for e.g. PAL) IvdL
site_incalt = {} # option to increase sampling altitude for sites specified in sites and weights file
for key, value in sites_weights.iteritems():
for key, value in sites_weights.items():
if 'co2_' in key or 'sf6' in key: # to be fixed later, do not yet know how to parse valid keys from rc-files yet.... WP
sitename, sitecategory = key, value
sitename = sitename.strip()
......@@ -350,8 +350,8 @@ class ObsPackObservations(Observations):
identifier = obs.code
species, site, method, lab, datasetnr = identifier.split('_')
if site_info.has_key(identifier):
if site_hourly.has_key(identifier):
if identifier in site_info:
if identifier in site_hourly:
obs.samplingstrategy = 2
hourf, hourt = site_hourly[identifier]
if int(obs.xdate.hour) >= hourf and int(obs.xdate.hour) <= hourt:
......@@ -375,7 +375,7 @@ class ObsPackObservations(Observations):
else:
logging.warning("Observation NOT found (%s, %d), please check sites.rc file (%s) !!!" % (identifier, obs.id, self.sites_file))
if site_move.has_key(identifier):
if identifier in site_move:
movelat, movelon = site_move[identifier]
obs.lat = obs.lat + movelat
......@@ -383,7 +383,7 @@ class ObsPackObservations(Observations):
logging.warning("Observation location for (%s, %d), is moved by %3.2f degrees latitude and %3.2f degrees longitude" % (identifier, obs.id, movelat, movelon))
if site_incalt.has_key(identifier):
if identifier in site_incalt:
incalt = site_incalt[identifier]
obs.height = obs.height + incalt
......@@ -418,7 +418,7 @@ class ObsPackObservations(Observations):
f.close()
#return outfile
for key, value in self.site_move.iteritems():
for key, value in self.site_move.items():
msg = "Site is moved by %3.2f degrees latitude and %3.2f degrees longitude" % value
f.add_attribute(key, msg)
......
......@@ -25,7 +25,7 @@ import sys
import logging
import datetime as dtm
from string import strip
#from string import strip
from numpy import array, logical_and, sqrt
sys.path.append(os.getcwd())
sys.path.append('../../')
......@@ -52,7 +52,7 @@ class ObsPackObservations(Observations):
if not os.path.exists(op_dir):
msg = 'Could not find the required ObsPack distribution (%s) ' % op_dir
logging.error(msg)
raise IOError, msg
raise IOError(msg)
else:
self.obspack_dir = op_dir
self.obspack_id = op_id
......@@ -107,7 +107,7 @@ class ObsPackObservations(Observations):
else:
obspackid = ncf.get_variable('obspack_id').take(subselect, axis=0)
obspackid = [s.tostring().lower() for s in obspackid]
obspackid = map(strip, obspackid)
obspackid = list(map(str.strip,str(obspackid)))
datasetname = ncfile # use full name of dataset to propagate for clarity
lats = ncf.get_variable('latitude').take(subselect, axis=0)
lons = ncf.get_variable('longitude').take(subselect, axis=0)
......@@ -133,7 +133,7 @@ class ObsPackObservations(Observations):
logging.error(msg)
logging.error("Did the sampling step succeed?")
logging.error("...exiting")
raise IOError, msg
raise IOError(msg)
ncf = io.ct_read(filename, method='read')
ids = ncf.get_variable('obs_num')
......@@ -142,7 +142,7 @@ class ObsPackObservations(Observations):
logging.info("Successfully read data from model sample file (%s)" % filename)
obs_ids = self.getvalues('id').tolist()
ids = map(int, ids)
ids = list(map(int, ids))
missing_samples = []
......@@ -179,7 +179,7 @@ class ObsPackObservations(Observations):
dim10char = f.add_dim('string_of10chars', 10)
dimcalcomp = f.add_dim('calendar_components', 6)
for key, value in self.site_move.iteritems():
for key, value in self.site_move.items():
msg = "Site is moved by %3.2f degrees latitude and %3.2f degrees longitude" % value
f.add_attribute(key, msg)
......@@ -302,7 +302,7 @@ class ObsPackObservations(Observations):
if not os.path.exists(filename):
msg = 'Could not find the required sites.rc input file (%s) ' % filename
logging.error(msg)
raise IOError, msg
raise IOError(msg)
else:
self.sites_file = filename
......@@ -330,7 +330,7 @@ class ObsPackObservations(Observations):
site_info = {}
site_move = {}
site_incalt = {} # option to increase sampling altitude for sites specified in sites and weights file
for key, value in sites_weights.iteritems():
for key, value in sites_weights.items():
if 'co2_' in key or 'sf6' in key: # to be fixed later, do not yet know how to parse valid keys from rc-files yet.... WP
sitename, sitecategory = key, value
sitename = sitename.strip()
......@@ -357,7 +357,7 @@ class ObsPackObservations(Observations):
identifier = obs.code
species, site, method, lab, datasetnr = identifier.split('_')
if site_info.has_key(identifier):
if identifier in site_info:
if site_info[identifier]['category'] == 'do-not-use' or obs.flag == 99:
logging.warning("Observation found (%s, %d), but not used in assimilation." % (identifier, obs.id))
obs.mdm = site_info[identifier]['error'] * self.global_R_scaling
......@@ -377,7 +377,7 @@ class ObsPackObservations(Observations):
else:
logging.warning("Observation NOT found (%s, %d), please check sites.rc file (%s) !!!" % (identifier, obs.id, self.sites_file))
if site_move.has_key(identifier):
if identifier in site_move:
movelat, movelon = site_move[identifier]
obs.lat = obs.lat + movelat
......@@ -385,7 +385,7 @@ class ObsPackObservations(Observations):
logging.warning("Observation location for (%s, %d), is moved by %3.2f degrees latitude and %3.2f degrees longitude" % (identifier, obs.id, movelat, movelon))
if site_incalt.has_key(identifier):
if identifier in site_incalt:
incalt = site_incalt[identifier]
obs.height = obs.height + incalt
......@@ -419,7 +419,7 @@ class ObsPackObservations(Observations):
f.close()
#return outfile
for key, value in self.site_move.iteritems():
for key, value in self.site_move.items():
msg = "Site is moved by %3.2f degrees latitude and %3.2f degrees longitude" % value
f.add_attribute(key, msg)
......
......@@ -25,7 +25,7 @@ import sys
import logging
import datetime as dtm
from string import strip
#from string import strip
from numpy import array, logical_and, sqrt
sys.path.append(os.getcwd())
sys.path.append('../../')
......@@ -52,7 +52,7 @@ class ObsPackObservations(Observations):
if not os.path.exists(op_dir):
msg = 'Could not find the required ObsPack distribution (%s) ' % op_dir
logging.error(msg)
raise IOError, msg
raise IOError(msg)
else:
self.obspack_dir = op_dir
self.obspack_id = op_id
......@@ -107,7 +107,7 @@ class ObsPackObservations(Observations):
else:
obspackid = ncf.get_variable('obspack_id').take(subselect, axis=0)
obspackid = [s.tostring().lower() for s in obspackid]
obspackid = map(strip, obspackid)
obspackid = list(map(str.strip,str(obspackid)))
datasetname = ncfile # use full name of dataset to propagate for clarity
lats = ncf.get_variable('latitude').take(subselect, axis=0)
lons = ncf.get_variable('longitude').take(subselect, axis=0)
......@@ -133,7 +133,7 @@ class ObsPackObservations(Observations):
logging.error(msg)
logging.error("Did the sampling step succeed?")
logging.error("...exiting")
raise IOError, msg
raise IOError(msg)
ncf = io.ct_read(filename, method='read')
ids = ncf.get_variable('obs_num')
......@@ -142,7 +142,7 @@ class ObsPackObservations(Observations):
logging.info("Successfully read data from model sample file (%s)" % filename)
obs_ids = self.getvalues('id').tolist()
ids = map(int, ids)
ids = list(map(int, ids))
missing_samples = []
......@@ -179,7 +179,7 @@ class ObsPackObservations(Observations):
dim10char = f.add_dim('string_of10chars', 10)
dimcalcomp = f.add_dim('calendar_components', 6)
for key, value in self.site_move.iteritems():
for key, value in self.site_move.items():
msg = "Site is moved by %3.2f degrees latitude and %3.2f degrees longitude" % value
f.add_attribute(key, msg)
......@@ -302,7 +302,7 @@ class ObsPackObservations(Observations):
if not os.path.exists(filename):
msg = 'Could not find the required sites.rc input file (%s) ' % filename
logging.error(msg)
raise IOError, msg
raise IOError(msg)
else:
self.sites_file = filename
......@@ -330,7 +330,7 @@ class ObsPackObservations(Observations):
site_info = {}
site_move = {}
site_incalt = {} # option to increase sampling altitude for sites specified in sites and weights file
for key, value in sites_weights.iteritems():
for key, value in sites_weights.items():
if 'co2_' in key or 'sf6' in key: # to be fixed later, do not yet know how to parse valid keys from rc-files yet.... WP
sitename, sitecategory = key, value
sitename = sitename.strip()
......@@ -357,7 +357,7 @@ class ObsPackObservations(Observations):
identifier = obs.code
species, site, method, lab, datasetnr = identifier.split('_')
if site_info.has_key(identifier):
if identifier in site_info:
if site_info[identifier]['category'] == 'do-not-use' or obs.flag == 99:
logging.warning("Observation found (%s, %d), but not used in assimilation." % (identifier, obs.id))
obs.mdm = site_info[identifier]['error'] * self.global_R_scaling
......@@ -377,7 +377,7 @@ class ObsPackObservations(Observations):
else:
logging.warning("Observation NOT found (%s, %d), please check sites.rc file (%s) !!!" % (identifier, obs.id, self.sites_file))
if site_move.has_key(identifier):
if identifier in site_move:
movelat, movelon = site_move[identifier]
obs.lat = obs.lat + movelat
......@@ -385,7 +385,7 @@ class ObsPackObservations(Observations):
logging.warning("Observation location for (%s, %d), is moved by %3.2f degrees latitude and %3.2f degrees longitude" % (identifier, obs.id, movelat, movelon))
if site_incalt.has_key(identifier):
if identifier in site_incalt:
incalt = site_incalt[identifier]
obs.height = obs.height + incalt
......@@ -419,7 +419,7 @@ class ObsPackObservations(Observations):
f.close()
#return outfile
for key, value in self.site_move.iteritems():
for key, value in self.site_move.items():
msg = "Site is moved by %3.2f degrees latitude and %3.2f degrees longitude" % value
f.add_attribute(key, msg)
......
......@@ -62,7 +62,7 @@ class CO2StateVector(StateVector):
if not os.path.exists(fil):
msg = "Cannot find the specified file %s" % fil
logging.error(msg)
raise IOError, msg
raise IOError(msg)
else:
logging.info("Using covariance file: %s" % fil)
......@@ -70,7 +70,7 @@ class CO2StateVector(StateVector):
f_bio = io.ct_read(file_bio_cov, 'read')
cov_ocn = f_ocn.get_variable('CORMAT')
if f_bio.variables.has_key('covariance'):
if 'covariance' in f_bio.variables:
cov_bio = f_bio.get_variable('covariance') # newly created CTDAS covariance files
else:
cov_bio = f_bio.get_variable('qprior') # old CarbonTracker covariance files
......
......@@ -61,10 +61,10 @@ class CO2GriddedDaSystem(DaSystem):
if v == 'False': self[k] = False
for key in needed_rc_items:
if not self.has_key(key):
if key not in self:
msg = 'Missing a required value in rc-file : %s' % key
logging.error(msg)
raise IOError, msg
raise IOError(msg)
logging.debug('DA System Info settings have been validated succesfully')
......
......@@ -71,7 +71,7 @@ class CO2GriddedStateVector(StateVector):
if not os.path.exists(file):
msg = "Cannot find the specified file %s" % file
logging.error(msg)
raise IOError, msg
raise IOError(msg)
else:
logging.debug("Using covariance file: %s" % file)
......
......@@ -81,7 +81,7 @@ class CO2GriddedStateVector(StateVector):
if not os.path.exists(file):
msg = "Cannot find the specified file %s" % file
logging.error(msg)
raise IOError, msg
raise IOError(msg)
else:
logging.debug("Using covariance file: %s" % file)