From d5400290a8168bee0dfb7302eef29d5f3d97a11f Mon Sep 17 00:00:00 2001
From: Aki Tsuruta <tsurutaa@puhti-login1.bullx>
Date: Thu, 29 Oct 2020 13:16:54 +0200
Subject: [PATCH] update on methane, mainly changes due to Python2->Python3
---
da/methane/initexit.py | 92 ++++++++++++++++--------------
da/methane/io4.py | 18 +++---
da/methane/obs.py | 34 +++++------
da/methane/statevector.py | 12 ++--
da/methane/statevector_2lambdas.py | 16 +++---
5 files changed, 85 insertions(+), 87 deletions(-)
diff --git a/da/methane/initexit.py b/da/methane/initexit.py
index 9079cbff..0865efb0 100755
--- a/da/methane/initexit.py
+++ b/da/methane/initexit.py
@@ -1,8 +1,8 @@
-"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters.
-Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
-updates of the code. See also: http://www.carbontracker.eu.
-
-This program is free software: you can redistribute it and/or modify it under the
+"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters.
+Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
+updates of the code. See also: http://www.carbontracker.eu.
+
+This program is free software: you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software Foundation,
version 3. This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
@@ -24,7 +24,7 @@ import logging
import os
import sys
import numpy as np
-from string import join
+#from string import join
import da.tools.rc as rc
from da.tools.initexit import *
@@ -40,35 +40,40 @@ needed_da_items = [
'da.obsoperator',
'da.obsoperator.rc',
'da.optimizer.nmembers',
- 'da.suffixname']
-
-
-#def validate_rc2(self):
-# """
-# Validate the contents of the rc-file given a dictionary of required keys.
-# Currently required keys are :attr:`~da.tools.initexit.needed_da_items`
-# """
-#
-# for k, v in self.iteritems():
-# if v in ['True', 'true', 't', 'T', 'y', 'yes']:
-# self[k] = True
-# if v in ['False', 'false', 'f', 'F', 'n', 'no']:
-# self[k] = False
-# if 'date' in k :
-# self[k] = to_datetime(v)
-# if k in ['time.start', 'time.end', 'time.finish', 'da.restart.tstamp']:
-# self[k] = to_datetime(v)
-# for key in needed_da_items:
-# if not self.has_key(key):
-# msg = 'Missing a required value in rc-file : %s' % key
-# logging.error(msg)
-# logging.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ')
-# logging.error('Please note the update on Dec 02 2011 where rc-file names for DaSystem and ')
-# logging.error('are from now on specified in the main rc-file (see da/rc/da.rc for example)')
-# logging.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ')
-# raise IOError, msg
-# logging.debug('DA Cycle settings have been validated succesfully')
-#CycleControl.validate_rc = validate_rc2
+ 'da.suffixname',
+ 'da.resources.ntime',
+ 'da.resources.npes',
+ 'da.projectaccount',
+ 'da.tm5.jobfilename'
+]
+
+
+def validate_rc2(self):
+ """
+ Validate the contents of the rc-file given a dictionary of required keys.
+ Currently required keys are :attr:`~da.tools.initexit.needed_da_items`
+ """
+
+ for k, v in self.items():
+ if v in ['True', 'true', 't', 'T', 'y', 'yes']:
+ self[k] = True
+ if v in ['False', 'false', 'f', 'F', 'n', 'no']:
+ self[k] = False
+ if 'date' in k :
+ self[k] = to_datetime(v)
+ if k in ['time.start', 'time.end', 'time.finish', 'da.restart.tstamp']:
+ self[k] = to_datetime(v)
+ for key in needed_da_items:
+ if key not in self:
+ msg = 'Missing a required value in rc-file : %s' % key
+ logging.error(msg)
+ logging.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ')
+ logging.error('Please note the update on Dec 02 2011 where rc-file names for DaSystem and ')
+ logging.error('are from now on specified in the main rc-file (see da/rc/da.rc for example)')
+ logging.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ')
+ raise IOError(msg)
+ logging.debug('DA Cycle settings have been validated succesfully')
+CycleControl.validate_rc = validate_rc2
def submit_next_cycle_fmi(self):
"""
@@ -88,21 +93,22 @@ def submit_next_cycle_fmi(self):
jobid = self['time.end'].strftime('%Y%m%d')
targetdir = os.path.join(self['dir.exec'])
jobfile = os.path.join(targetdir, 'jb.%s.jb' % jobid)
- logfile = os.path.join(targetdir, 'jb.%s.log' % jobid)
# Template and commands for job
- jobparams = {'jobname':"ctdas", 'jobtime':'05:00:00', 'logfile': logfile, 'errfile': logfile,
- 'jobpes':'120','jobnodes':'20'}
+ jobtime = self['da.resources.ntime']
+ jobprojectaccount = self['da.projectaccount']
+ jobparams = {'jobname':"ctdas", 'jobtime':jobtime, 'jobnpes':'1', 'projectaccount':jobprojectaccount}
template = self.daplatform.get_job_template(jobparams)
execcommand = os.path.join(self['dir.da_submit'], sys.argv[0])
if '-t' in self.opts:
(self.opts).remove('-t')
- #template += 'python %s rc=%s %s' % (execcommand, self['da.restart.fname'], join(self.opts, ''))
+ template += 'rcfile=%s \n'%self['da.restart.fname']
template += 'cd %s \n' %self['dir.da_submit']
- template += '%s rc=%s %s' % (sys.argv[0], self['da.restart.fname'], join(self.opts, ''))
+ template += 'python %s rc=$rcfile %s \n' % (sys.argv[0], ''.join(self.opts))
+ template += './save2allas/ctdas.sh $rcfile'
# write and submit
self.daplatform.write_job(jobfile, template, jobid)
- jobid = self.daplatform.submit_job(jobfile, joblog=logfile)
- job_file = 'ctdas.o%s' %jobid[0:-4]
+ jobid = self.daplatform.submit_job(jobfile)
+ job_file = 'ctdas.o%s' %jobid #[0:-4]
logging.info('Jobfile %s' %job_file )
logging.info('DA cycle has been submitted.')
@@ -122,7 +128,6 @@ def setup_file_structure_fmi(self):
self['dir.input'] = os.path.join(self['dir.da_run'], 'input_%s' %suf )
self['dir.output'] = os.path.join(self['dir.da_run'], 'output_%s' %suf )
self['dir.analysis'] = os.path.join(self['dir.da_run'], 'analysis_%s' %suf )
- #self['dir.jobs'] = os.path.join(self['dir.da_run'], 'jobs')
self['dir.restart'] = os.path.join(self['dir.da_run'], 'restart_%s' %suf )
logging.info("setup_file_structure %s" %self['dir.output'])
@@ -131,7 +136,6 @@ def setup_file_structure_fmi(self):
create_dirs(os.path.join(self['dir.input']))
create_dirs(os.path.join(self['dir.output']))
create_dirs(os.path.join(self['dir.analysis']))
- #create_dirs(os.path.join(self['dir.jobs']))
create_dirs(os.path.join(self['dir.restart']))
logging.info('Succesfully created the file structure for the assimilation job')
diff --git a/da/methane/io4.py b/da/methane/io4.py
index 39104d2d..0cac13a0 100755
--- a/da/methane/io4.py
+++ b/da/methane/io4.py
@@ -1,8 +1,8 @@
-"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters.
-Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
-updates of the code. See also: http://www.carbontracker.eu.
-
-This program is free software: you can redistribute it and/or modify it under the
+"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters.
+Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
+updates of the code. See also: http://www.carbontracker.eu.
+
+This program is free software: you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software Foundation,
version 3. This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
@@ -20,18 +20,16 @@ Revision History:
File created on Apr 2016.
"""
-from . import standardvariables
import datetime as dt
from numpy import array, arange
import os
import logging
import sys
-sys.path.append('/stornext/store/carbon/CarbonTracker/netcdf4/python_packages_install_dir/lib/python2.7/site-packages/')
import netCDF4
-sys.path.append('/stornext/store/carbon/CarbonTracker/pyhdf/pyhdf-0.8.3/lib/python2.7/site-packages/')
import pyhdf.SD as hdf
sys.path.append('../')
from da.tools.io4 import *
+import da.methane.standardvariables
disclaimer = "This data belongs to the CarbonTracker project"
@@ -56,9 +54,9 @@ def add_tc_header2(self):
def standard_var2(self,varname):
""" return properties of standard variables """
- from . import standardvariables
+ import da.methane.standardvariables as standardvariables
- if varname in list(standardvariables.standard_variables.keys()):
+ if varname in standardvariables.standard_variables.keys():
return standardvariables.standard_variables[varname]
else:
return standardvariables.standard_variables['unknown']
diff --git a/da/methane/obs.py b/da/methane/obs.py
index 5f098445..0359bdf6 100755
--- a/da/methane/obs.py
+++ b/da/methane/obs.py
@@ -1,8 +1,8 @@
-"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters.
-Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
-updates of the code. See also: http://www.carbontracker.eu.
-
-This program is free software: you can redistribute it and/or modify it under the
+"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters.
+Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
+updates of the code. See also: http://www.carbontracker.eu.
+
+This program is free software: you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software Foundation,
version 3. This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
@@ -14,18 +14,16 @@ program. If not, see <http://www.gnu.org/licenses/>."""
# obs.py
"""
-Author : aki
+Author : peters
Revision History:
-File revised on Feb 2017.
+File created on 28 Jul 2010.
"""
import os
import sys
import logging
-#from da.baseclasses.statevector import filename
import datetime as dtm
-from string import strip
from numpy import array, logical_and
sys.path.append(os.getcwd())
@@ -86,8 +84,8 @@ class MethaneObservations(Observations):
#evn = [s.tostring().lower() for s in evn]
#evn = map(strip, evn)
sites = ncf.get_variable('obs_id').take(subselect, axis=0)
- sites = [s.tostring().lower() for s in sites]
- sites = list(map(strip, sites))
+ sites = [s.tostring().lower().decode("utf-8") for s in sites]
+ sites = list(map(str.strip, sites))
lats = ncf.get_variable('latitude').take(subselect, axis=0)
lons = ncf.get_variable('longitude').take(subselect, axis=0)
alts = ncf.get_variable('altitude').take(subselect, axis=0)
@@ -107,7 +105,6 @@ class MethaneObservations(Observations):
for n in range(len(dates)):
obs[n] = obs[n]
- #self.datalist.append(MoleFractionSample(ids[n], dates[n], sites[n], obs[n], 0.0, 0.0, 0.0, 0.0, flags[n], alts[n], lats[n], lons[n], evn[n], species[n], strategy[n], 0.0, self.obs_filename))
self.datalist.append( MoleFractionSample(ids[n], dates[n], sites[n], obs[n], 0.0, 0.0, 0.0, 0.0, 0.0, alts[n], lats[n], lons[n], '0000', 'ch4', 1.0, 0.0, self.obs_filename) )
logging.debug("Added %d observations to the Data list" % len(dates))
@@ -125,8 +122,7 @@ class MethaneObservations(Observations):
ncf = io.ct_read(filename, method='read')
ids = ncf.get_variable('obs_num')
simulated = ncf.get_variable('flask')
- #for i in xrange(simulated.shape[0]):
- # print simulated[i,:]
+
ncf.close()
logging.info("Successfully read data from model sample file (%s)" % filename)
@@ -157,7 +153,7 @@ class MethaneObservations(Observations):
"""
f = io.CT_CDF(obsinputfile, method='create')
- logging.debug('Creating new observations file for ObservationOperator (%s)' % obsinputfile)
+ logging.info('Creating new observations file for ObservationOperator (%s)' % obsinputfile)
dimid = f.add_dim('obs', len(self.datalist))
dim200char = f.add_dim('string_of200chars', 200)
@@ -168,7 +164,7 @@ class MethaneObservations(Observations):
#return obsinputfile
data = self.getvalues('id')
-
+
savedict = io.std_savedict.copy()
savedict['name'] = "obs_num"
savedict['dtype'] = "int"
@@ -284,7 +280,7 @@ class MethaneObservations(Observations):
logging.debug('Model-data mismatch active sites : %d ' % self.n_sites_active)
logging.debug('Model-data mismatch moved sites : %d ' % self.n_sites_moved)
- cats = [k for k in list(sites_weights.keys()) if 'site.category' in k]
+ cats = [k for k in sites_weights.keys() if 'site.category' in k]
SiteCategories = {}
for key in cats:
@@ -296,7 +292,7 @@ class MethaneObservations(Observations):
SiteCategories[name] = {'error':error, 'may_localize':may_localize, 'may_reject':may_reject}
#print name,SiteCategories[name]
- active = [k for k in list(sites_weights.keys()) if 'site.active' in k]
+ active = [k for k in sites_weights.keys() if 'site.active' in k]
site_info = {}
for key in active:
@@ -308,7 +304,7 @@ class MethaneObservations(Observations):
for obs in self.datalist:
obs.mdm = 1000.0 # default is very high model-data-mismatch, until explicitly set by script
- if obs.code in site_info:
+ if obs.code in site_info:
logging.debug("Observation found (%s)" % obs.code)
obs.mdm = site_info[obs.code]['error'] * self.global_R_scaling
obs.may_localize = site_info[obs.code]['may_localize']
diff --git a/da/methane/statevector.py b/da/methane/statevector.py
index b75dafce..df0b0b65 100755
--- a/da/methane/statevector.py
+++ b/da/methane/statevector.py
@@ -1,8 +1,8 @@
-"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters.
-Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
-updates of the code. See also: http://www.carbontracker.eu.
-
-This program is free software: you can redistribute it and/or modify it under the
+"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters.
+Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
+updates of the code. See also: http://www.carbontracker.eu.
+
+This program is free software: you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software Foundation,
version 3. This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
@@ -43,7 +43,7 @@ class MethaneStateVector(StateVector):
self.speciesdict = {'ch4': np.ones(self.nparams)}
logging.debug("A species mask was created, only the following species are recognized in this system:")
- for k in list(self.speciesdict.keys()):
+ for k in self.speciesdict.keys():
logging.debug(" -> %s" % k)
def get_covariance(self, date, dacycle):
diff --git a/da/methane/statevector_2lambdas.py b/da/methane/statevector_2lambdas.py
index 6582cdd9..2c467996 100755
--- a/da/methane/statevector_2lambdas.py
+++ b/da/methane/statevector_2lambdas.py
@@ -1,8 +1,8 @@
-"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters.
-Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
-updates of the code. See also: http://www.carbontracker.eu.
-
-This program is free software: you can redistribute it and/or modify it under the
+"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters.
+Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
+updates of the code. See also: http://www.carbontracker.eu.
+
+This program is free software: you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software Foundation,
version 3. This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
@@ -61,7 +61,7 @@ class MethaneStateVector(StateVector):
# These list objects hold the data for each time step of lag in the system. Note that the ensembles for each time step consist
# of lists of EnsembleMember objects, we define member 0 as the mean of the distribution and n=1,...,nmembers as the spread.
- self.ensemble_members = list(range(self.nlag))
+ self.ensemble_members = range(self.nlag)
for n in range(self.nlag):
self.ensemble_members[n] = []
@@ -119,7 +119,7 @@ class MethaneStateVector(StateVector):
self.speciesdict = {'ch4': np.ones(self.nparams)}
logging.debug("A species mask was created, only the following species are recognized in this system:")
- for k in list(self.speciesdict.keys()):
+ for k in self.speciesdict.keys():
logging.debug(" -> %s" % k)
def get_covariance(self, date, dacycle):
@@ -135,7 +135,7 @@ class MethaneStateVector(StateVector):
#----Arbitray covariance matrix ----#
#fullcov = np.zeros((self.nparams, self.nparams), float)
- #for i in xrange(self.nparams):
+ #for i in range(self.nparams):
# #fullcov[i,i] = 1. # Identity matrix
# fullcov[i,i] = 0.08
#fullcov[self.nparams-1,self.nparams-1] = 1.e-10
--
GitLab