Merge branch 'aki' into 'master'

update on methane, mainly changes due to Python2->Python3

See merge request !20

da/methane/initexit.py

-"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters. 
-Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
-updates of the code. See also: http://www.carbontracker.eu. 
-
-This program is free software: you can redistribute it and/or modify it under the
+"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters.  
+Users are recommended to contact the developers (wouter.peters@wur.nl) to receive 
+updates of the code. See also: http://www.carbontracker.eu.  
+ 
+This program is free software: you can redistribute it and/or modify it under the 
 terms of the GNU General Public License as published by the Free Software Foundation, 
 version 3. This program is distributed in the hope that it will be useful, but 
 WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
@@ -24,7 +24,7 @@ import logging
 import os
 import sys
 import numpy as np
-from string import join
+#from string import join
 
 import da.tools.rc as rc
 from da.tools.initexit import *
@@ -40,35 +40,40 @@ needed_da_items = [
     'da.obsoperator',
     'da.obsoperator.rc',
     'da.optimizer.nmembers',
-    'da.suffixname']
-
-
-#def validate_rc2(self):
-#    """ 
-#    Validate the contents of the rc-file given a dictionary of required keys. 
-#    Currently required keys are :attr:`~da.tools.initexit.needed_da_items`
-#    """
-#
-#    for k, v in self.iteritems():
-#        if v in ['True', 'true', 't', 'T', 'y', 'yes']:
-#            self[k] = True
-#        if v in ['False', 'false', 'f', 'F', 'n', 'no']:
-#            self[k] = False
-#        if 'date' in k : 
-#            self[k] = to_datetime(v)
-#        if k in ['time.start', 'time.end', 'time.finish', 'da.restart.tstamp']:
-#            self[k] = to_datetime(v)
-#    for key in needed_da_items:
-#        if not self.has_key(key):
-#            msg = 'Missing a required value in rc-file : %s' % key
-#            logging.error(msg)
-#            logging.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ')
-#            logging.error('Please note the update on Dec 02 2011 where rc-file names for DaSystem and ')
-#            logging.error('are from now on specified in the main rc-file (see da/rc/da.rc for example)')
-#            logging.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ')
-#            raise IOError, msg
-#    logging.debug('DA Cycle settings have been validated succesfully')
-#CycleControl.validate_rc = validate_rc2
+    'da.suffixname',
+    'da.resources.ntime',
+    'da.resources.npes',
+    'da.projectaccount',
+    'da.tm5.jobfilename'
+]
+
+
+def validate_rc2(self):
+    """ 
+    Validate the contents of the rc-file given a dictionary of required keys. 
+    Currently required keys are :attr:`~da.tools.initexit.needed_da_items`
+    """
+
+    for k, v in self.items():
+        if v in ['True', 'true', 't', 'T', 'y', 'yes']:
+            self[k] = True
+        if v in ['False', 'false', 'f', 'F', 'n', 'no']:
+            self[k] = False
+        if 'date' in k : 
+            self[k] = to_datetime(v)
+        if k in ['time.start', 'time.end', 'time.finish', 'da.restart.tstamp']:
+            self[k] = to_datetime(v)
+    for key in needed_da_items:
+        if key not in self:
+            msg = 'Missing a required value in rc-file : %s' % key
+            logging.error(msg)
+            logging.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ')
+            logging.error('Please note the update on Dec 02 2011 where rc-file names for DaSystem and ')
+            logging.error('are from now on specified in the main rc-file (see da/rc/da.rc for example)')
+            logging.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ')
+            raise IOError(msg)
+    logging.debug('DA Cycle settings have been validated succesfully')
+CycleControl.validate_rc = validate_rc2
 
 def submit_next_cycle_fmi(self):
     """ 
@@ -88,21 +93,22 @@ def submit_next_cycle_fmi(self):
         jobid = self['time.end'].strftime('%Y%m%d') 
         targetdir = os.path.join(self['dir.exec'])
         jobfile = os.path.join(targetdir, 'jb.%s.jb' % jobid)
-        logfile = os.path.join(targetdir, 'jb.%s.log' % jobid)
         # Template and commands for job
-        jobparams = {'jobname':"ctdas", 'jobtime':'05:00:00', 'logfile': logfile, 'errfile': logfile,
-                      'jobpes':'120','jobnodes':'20'}
+        jobtime  = self['da.resources.ntime']
+        jobprojectaccount   = self['da.projectaccount']
+        jobparams = {'jobname':"ctdas", 'jobtime':jobtime, 'jobnpes':'1', 'projectaccount':jobprojectaccount}
         template = self.daplatform.get_job_template(jobparams)
         execcommand = os.path.join(self['dir.da_submit'], sys.argv[0]) 
         if '-t' in self.opts:
             (self.opts).remove('-t') 
-        #template += 'python %s rc=%s %s' % (execcommand, self['da.restart.fname'], join(self.opts, '')) 
+        template += 'rcfile=%s \n'%self['da.restart.fname']
         template += 'cd %s \n' %self['dir.da_submit']
-        template += '%s rc=%s %s' % (sys.argv[0], self['da.restart.fname'], join(self.opts, '')) 
+        template += 'python %s rc=$rcfile %s \n' % (sys.argv[0], ''.join(self.opts)) 
+        template += './save2allas/ctdas.sh $rcfile'
         # write and submit 
         self.daplatform.write_job(jobfile, template, jobid)
-        jobid = self.daplatform.submit_job(jobfile, joblog=logfile) 
-        job_file = 'ctdas.o%s' %jobid[0:-4]
+        jobid = self.daplatform.submit_job(jobfile) 
+        job_file = 'ctdas.o%s' %jobid #[0:-4]
         logging.info('Jobfile %s' %job_file )
         logging.info('DA cycle has been submitted.')
 
@@ -122,7 +128,6 @@ def setup_file_structure_fmi(self):
     self['dir.input'] = os.path.join(self['dir.da_run'], 'input_%s' %suf )
     self['dir.output'] = os.path.join(self['dir.da_run'], 'output_%s' %suf )
     self['dir.analysis'] = os.path.join(self['dir.da_run'], 'analysis_%s' %suf )
-    #self['dir.jobs'] = os.path.join(self['dir.da_run'], 'jobs')
     self['dir.restart'] = os.path.join(self['dir.da_run'], 'restart_%s' %suf )
 
     logging.info("setup_file_structure %s" %self['dir.output'])
@@ -131,7 +136,6 @@ def setup_file_structure_fmi(self):
     create_dirs(os.path.join(self['dir.input']))
     create_dirs(os.path.join(self['dir.output']))
     create_dirs(os.path.join(self['dir.analysis']))
-    #create_dirs(os.path.join(self['dir.jobs']))
     create_dirs(os.path.join(self['dir.restart']))
 
     logging.info('Succesfully created the file structure for the assimilation job')

da/methane/io4.py

-"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters. 
-Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
-updates of the code. See also: http://www.carbontracker.eu. 
-
-This program is free software: you can redistribute it and/or modify it under the
+"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters.  
+Users are recommended to contact the developers (wouter.peters@wur.nl) to receive 
+updates of the code. See also: http://www.carbontracker.eu.  
+ 
+This program is free software: you can redistribute it and/or modify it under the 
 terms of the GNU General Public License as published by the Free Software Foundation, 
 version 3. This program is distributed in the hope that it will be useful, but 
 WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
@@ -20,18 +20,16 @@ Revision History:
 File created on Apr 2016.
 
 """
-from . import standardvariables
 import datetime as dt
 from numpy import array, arange
 import os
 import logging
 import sys
-sys.path.append('/stornext/store/carbon/CarbonTracker/netcdf4/python_packages_install_dir/lib/python2.7/site-packages/')
 import netCDF4
-sys.path.append('/stornext/store/carbon/CarbonTracker/pyhdf/pyhdf-0.8.3/lib/python2.7/site-packages/')
 import pyhdf.SD as hdf
 sys.path.append('../')
 from da.tools.io4 import *
+import da.methane.standardvariables
 
 
 disclaimer = "This data belongs to the CarbonTracker project"
@@ -56,9 +54,9 @@ def add_tc_header2(self):
 
 def standard_var2(self,varname):
     """ return properties of standard variables """
-    from . import standardvariables
+    import da.methane.standardvariables as standardvariables
     
-    if varname in list(standardvariables.standard_variables.keys()):
+    if varname in standardvariables.standard_variables.keys():
         return standardvariables.standard_variables[varname]
     else:
         return standardvariables.standard_variables['unknown']

da/methane/obs.py

-"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters. 
-Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
-updates of the code. See also: http://www.carbontracker.eu. 
-
-This program is free software: you can redistribute it and/or modify it under the
+"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters.  
+Users are recommended to contact the developers (wouter.peters@wur.nl) to receive 
+updates of the code. See also: http://www.carbontracker.eu.  
+ 
+This program is free software: you can redistribute it and/or modify it under the 
 terms of the GNU General Public License as published by the Free Software Foundation, 
 version 3. This program is distributed in the hope that it will be useful, but 
 WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
@@ -14,18 +14,16 @@ program. If not, see <http://www.gnu.org/licenses/>."""
 # obs.py
 
 """
-Author : aki 
+Author : peters 
 
 Revision History:
-File revised on Feb 2017.
+File created on 28 Jul 2010.
 
 """
 import os
 import sys
 import logging
-#from da.baseclasses.statevector import filename
 import datetime as dtm
-from string import strip
 from numpy import array, logical_and
 
 sys.path.append(os.getcwd())
@@ -86,8 +84,8 @@ class MethaneObservations(Observations):
         #evn = [s.tostring().lower() for s in evn]
         #evn = map(strip, evn)
         sites = ncf.get_variable('obs_id').take(subselect, axis=0)
-        sites = [s.tostring().lower() for s in sites]
-        sites = list(map(strip, sites))
+        sites = [s.tostring().lower().decode("utf-8") for s in sites]
+        sites = list(map(str.strip, sites))
         lats = ncf.get_variable('latitude').take(subselect, axis=0)
         lons = ncf.get_variable('longitude').take(subselect, axis=0)
         alts = ncf.get_variable('altitude').take(subselect, axis=0)
@@ -107,7 +105,6 @@ class MethaneObservations(Observations):
 
         for n in range(len(dates)): 
             obs[n] = obs[n]
-            #self.datalist.append(MoleFractionSample(ids[n], dates[n], sites[n], obs[n], 0.0, 0.0, 0.0, 0.0, flags[n], alts[n], lats[n], lons[n], evn[n], species[n], strategy[n], 0.0, self.obs_filename))
             self.datalist.append( MoleFractionSample(ids[n], dates[n], sites[n], obs[n], 0.0, 0.0, 0.0, 0.0, 0.0, alts[n], lats[n], lons[n], '0000', 'ch4', 1.0, 0.0, self.obs_filename) ) 
         logging.debug("Added %d observations to the Data list" % len(dates))
 
@@ -125,8 +122,7 @@ class MethaneObservations(Observations):
         ncf = io.ct_read(filename, method='read')
         ids = ncf.get_variable('obs_num')
         simulated = ncf.get_variable('flask')
-        #for i in xrange(simulated.shape[0]):
-        #    print simulated[i,:]
+
         ncf.close()
         logging.info("Successfully read data from model sample file (%s)" % filename)
 
@@ -157,7 +153,7 @@ class MethaneObservations(Observations):
 
         """
         f = io.CT_CDF(obsinputfile, method='create')
-        logging.debug('Creating new observations file for ObservationOperator (%s)' % obsinputfile)
+        logging.info('Creating new observations file for ObservationOperator (%s)' % obsinputfile)
 
         dimid = f.add_dim('obs', len(self.datalist))
         dim200char = f.add_dim('string_of200chars', 200)
@@ -168,7 +164,7 @@ class MethaneObservations(Observations):
             #return obsinputfile
 
         data = self.getvalues('id')
-
+       
         savedict = io.std_savedict.copy() 
         savedict['name'] = "obs_num"
         savedict['dtype'] = "int"
@@ -284,7 +280,7 @@ class MethaneObservations(Observations):
         logging.debug('Model-data mismatch active sites       : %d ' % self.n_sites_active)
         logging.debug('Model-data mismatch moved sites        : %d ' % self.n_sites_moved)
    
-        cats = [k for k in list(sites_weights.keys()) if 'site.category' in k] 
+        cats = [k for k in sites_weights.keys() if 'site.category' in k] 
 
         SiteCategories = {}
         for key in cats:
@@ -296,7 +292,7 @@ class MethaneObservations(Observations):
             SiteCategories[name] = {'error':error, 'may_localize':may_localize, 'may_reject':may_reject}
             #print name,SiteCategories[name]
 
-        active = [k for k in list(sites_weights.keys()) if 'site.active' in k] 
+        active = [k for k in sites_weights.keys() if 'site.active' in k] 
 
         site_info = {}
         for key in active:
@@ -308,7 +304,7 @@ class MethaneObservations(Observations):
 
         for obs in self.datalist:
             obs.mdm = 1000.0  # default is very high model-data-mismatch, until explicitly set by script
-            if obs.code in site_info:
+            if obs.code in site_info: 
                 logging.debug("Observation found (%s)" % obs.code)
                 obs.mdm = site_info[obs.code]['error'] * self.global_R_scaling
                 obs.may_localize = site_info[obs.code]['may_localize']

da/methane/statevector.py

-"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters. 
-Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
-updates of the code. See also: http://www.carbontracker.eu. 
-
-This program is free software: you can redistribute it and/or modify it under the
+"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters.  
+Users are recommended to contact the developers (wouter.peters@wur.nl) to receive 
+updates of the code. See also: http://www.carbontracker.eu.  
+ 
+This program is free software: you can redistribute it and/or modify it under the 
 terms of the GNU General Public License as published by the Free Software Foundation, 
 version 3. This program is distributed in the hope that it will be useful, but 
 WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
@@ -43,7 +43,7 @@ class MethaneStateVector(StateVector):
 
         self.speciesdict = {'ch4': np.ones(self.nparams)}
         logging.debug("A species mask was created, only the following species are recognized in this system:")
-        for k in list(self.speciesdict.keys()): 
+        for k in self.speciesdict.keys(): 
             logging.debug("   ->    %s" % k)
 
     def get_covariance(self, date, dacycle):

da/methane/statevector_2lambdas.py

-"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters. 
-Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
-updates of the code. See also: http://www.carbontracker.eu. 
-
-This program is free software: you can redistribute it and/or modify it under the
+"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters.  
+Users are recommended to contact the developers (wouter.peters@wur.nl) to receive 
+updates of the code. See also: http://www.carbontracker.eu.  
+ 
+This program is free software: you can redistribute it and/or modify it under the 
 terms of the GNU General Public License as published by the Free Software Foundation, 
 version 3. This program is distributed in the hope that it will be useful, but 
 WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
@@ -61,7 +61,7 @@ class MethaneStateVector(StateVector):
         # These list objects hold the data for each time step of lag in the system. Note that the ensembles for each time step consist 
         # of lists of EnsembleMember objects, we define member 0 as the mean of the distribution and n=1,...,nmembers as the spread.
 
-        self.ensemble_members = list(range(self.nlag))
+        self.ensemble_members = range(self.nlag)
 
         for n in range(self.nlag):
             self.ensemble_members[n] = []
@@ -119,7 +119,7 @@ class MethaneStateVector(StateVector):
 
         self.speciesdict = {'ch4': np.ones(self.nparams)}
         logging.debug("A species mask was created, only the following species are recognized in this system:")
-        for k in list(self.speciesdict.keys()): 
+        for k in self.speciesdict.keys(): 
             logging.debug("   ->    %s" % k)
 
     def get_covariance(self, date, dacycle):
@@ -135,7 +135,7 @@ class MethaneStateVector(StateVector):
         #----Arbitray covariance matrix ----#
         #fullcov = np.zeros((self.nparams, self.nparams), float)
 
-        #for i in xrange(self.nparams):
+        #for i in range(self.nparams):
         #    #fullcov[i,i] = 1.  # Identity matrix
         #    fullcov[i,i] = 0.08
         #fullcov[self.nparams-1,self.nparams-1] = 1.e-10