diff --git a/da/methane/initexit.py b/da/methane/initexit.py index 9079cbffc5559b47bd0b8d84a630555b794e31ae..0865efb00818390f2008abc03b97ca1e5a85724c 100755 --- a/da/methane/initexit.py +++ b/da/methane/initexit.py @@ -1,8 +1,8 @@ -"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters. -Users are recommended to contact the developers (wouter.peters@wur.nl) to receive -updates of the code. See also: http://www.carbontracker.eu. - -This program is free software: you can redistribute it and/or modify it under the +"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters. +Users are recommended to contact the developers (wouter.peters@wur.nl) to receive +updates of the code. See also: http://www.carbontracker.eu. + +This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 3. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS @@ -24,7 +24,7 @@ import logging import os import sys import numpy as np -from string import join +#from string import join import da.tools.rc as rc from da.tools.initexit import * @@ -40,35 +40,40 @@ needed_da_items = [ 'da.obsoperator', 'da.obsoperator.rc', 'da.optimizer.nmembers', - 'da.suffixname'] - - -#def validate_rc2(self): -# """ -# Validate the contents of the rc-file given a dictionary of required keys. -# Currently required keys are :attr:`~da.tools.initexit.needed_da_items` -# """ -# -# for k, v in self.iteritems(): -# if v in ['True', 'true', 't', 'T', 'y', 'yes']: -# self[k] = True -# if v in ['False', 'false', 'f', 'F', 'n', 'no']: -# self[k] = False -# if 'date' in k : -# self[k] = to_datetime(v) -# if k in ['time.start', 'time.end', 'time.finish', 'da.restart.tstamp']: -# self[k] = to_datetime(v) -# for key in needed_da_items: -# if not self.has_key(key): -# msg = 'Missing a required value in rc-file : %s' % key -# logging.error(msg) -# logging.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ') -# logging.error('Please note the update on Dec 02 2011 where rc-file names for DaSystem and ') -# logging.error('are from now on specified in the main rc-file (see da/rc/da.rc for example)') -# logging.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ') -# raise IOError, msg -# logging.debug('DA Cycle settings have been validated succesfully') -#CycleControl.validate_rc = validate_rc2 + 'da.suffixname', + 'da.resources.ntime', + 'da.resources.npes', + 'da.projectaccount', + 'da.tm5.jobfilename' +] + + +def validate_rc2(self): + """ + Validate the contents of the rc-file given a dictionary of required keys. + Currently required keys are :attr:`~da.tools.initexit.needed_da_items` + """ + + for k, v in self.items(): + if v in ['True', 'true', 't', 'T', 'y', 'yes']: + self[k] = True + if v in ['False', 'false', 'f', 'F', 'n', 'no']: + self[k] = False + if 'date' in k : + self[k] = to_datetime(v) + if k in ['time.start', 'time.end', 'time.finish', 'da.restart.tstamp']: + self[k] = to_datetime(v) + for key in needed_da_items: + if key not in self: + msg = 'Missing a required value in rc-file : %s' % key + logging.error(msg) + logging.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ') + logging.error('Please note the update on Dec 02 2011 where rc-file names for DaSystem and ') + logging.error('are from now on specified in the main rc-file (see da/rc/da.rc for example)') + logging.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ') + raise IOError(msg) + logging.debug('DA Cycle settings have been validated succesfully') +CycleControl.validate_rc = validate_rc2 def submit_next_cycle_fmi(self): """ @@ -88,21 +93,22 @@ def submit_next_cycle_fmi(self): jobid = self['time.end'].strftime('%Y%m%d') targetdir = os.path.join(self['dir.exec']) jobfile = os.path.join(targetdir, 'jb.%s.jb' % jobid) - logfile = os.path.join(targetdir, 'jb.%s.log' % jobid) # Template and commands for job - jobparams = {'jobname':"ctdas", 'jobtime':'05:00:00', 'logfile': logfile, 'errfile': logfile, - 'jobpes':'120','jobnodes':'20'} + jobtime = self['da.resources.ntime'] + jobprojectaccount = self['da.projectaccount'] + jobparams = {'jobname':"ctdas", 'jobtime':jobtime, 'jobnpes':'1', 'projectaccount':jobprojectaccount} template = self.daplatform.get_job_template(jobparams) execcommand = os.path.join(self['dir.da_submit'], sys.argv[0]) if '-t' in self.opts: (self.opts).remove('-t') - #template += 'python %s rc=%s %s' % (execcommand, self['da.restart.fname'], join(self.opts, '')) + template += 'rcfile=%s \n'%self['da.restart.fname'] template += 'cd %s \n' %self['dir.da_submit'] - template += '%s rc=%s %s' % (sys.argv[0], self['da.restart.fname'], join(self.opts, '')) + template += 'python %s rc=$rcfile %s \n' % (sys.argv[0], ''.join(self.opts)) + template += './save2allas/ctdas.sh $rcfile' # write and submit self.daplatform.write_job(jobfile, template, jobid) - jobid = self.daplatform.submit_job(jobfile, joblog=logfile) - job_file = 'ctdas.o%s' %jobid[0:-4] + jobid = self.daplatform.submit_job(jobfile) + job_file = 'ctdas.o%s' %jobid #[0:-4] logging.info('Jobfile %s' %job_file ) logging.info('DA cycle has been submitted.') @@ -122,7 +128,6 @@ def setup_file_structure_fmi(self): self['dir.input'] = os.path.join(self['dir.da_run'], 'input_%s' %suf ) self['dir.output'] = os.path.join(self['dir.da_run'], 'output_%s' %suf ) self['dir.analysis'] = os.path.join(self['dir.da_run'], 'analysis_%s' %suf ) - #self['dir.jobs'] = os.path.join(self['dir.da_run'], 'jobs') self['dir.restart'] = os.path.join(self['dir.da_run'], 'restart_%s' %suf ) logging.info("setup_file_structure %s" %self['dir.output']) @@ -131,7 +136,6 @@ def setup_file_structure_fmi(self): create_dirs(os.path.join(self['dir.input'])) create_dirs(os.path.join(self['dir.output'])) create_dirs(os.path.join(self['dir.analysis'])) - #create_dirs(os.path.join(self['dir.jobs'])) create_dirs(os.path.join(self['dir.restart'])) logging.info('Succesfully created the file structure for the assimilation job') diff --git a/da/methane/io4.py b/da/methane/io4.py index 39104d2d1a3fb85af60b559d99bc3ea07c34b589..0cac13a0ac34da96fc0d0f1653db96d12a7ba285 100755 --- a/da/methane/io4.py +++ b/da/methane/io4.py @@ -1,8 +1,8 @@ -"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters. -Users are recommended to contact the developers (wouter.peters@wur.nl) to receive -updates of the code. See also: http://www.carbontracker.eu. - -This program is free software: you can redistribute it and/or modify it under the +"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters. +Users are recommended to contact the developers (wouter.peters@wur.nl) to receive +updates of the code. See also: http://www.carbontracker.eu. + +This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 3. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS @@ -20,18 +20,16 @@ Revision History: File created on Apr 2016. """ -from . import standardvariables import datetime as dt from numpy import array, arange import os import logging import sys -sys.path.append('/stornext/store/carbon/CarbonTracker/netcdf4/python_packages_install_dir/lib/python2.7/site-packages/') import netCDF4 -sys.path.append('/stornext/store/carbon/CarbonTracker/pyhdf/pyhdf-0.8.3/lib/python2.7/site-packages/') import pyhdf.SD as hdf sys.path.append('../') from da.tools.io4 import * +import da.methane.standardvariables disclaimer = "This data belongs to the CarbonTracker project" @@ -56,9 +54,9 @@ def add_tc_header2(self): def standard_var2(self,varname): """ return properties of standard variables """ - from . import standardvariables + import da.methane.standardvariables as standardvariables - if varname in list(standardvariables.standard_variables.keys()): + if varname in standardvariables.standard_variables.keys(): return standardvariables.standard_variables[varname] else: return standardvariables.standard_variables['unknown'] diff --git a/da/methane/obs.py b/da/methane/obs.py index 5f098445e53068f8e9b2f93017a97d1b42ab1158..0359bdf61d519e2b3c4d653e8b116f0d4bb3a321 100755 --- a/da/methane/obs.py +++ b/da/methane/obs.py @@ -1,8 +1,8 @@ -"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters. -Users are recommended to contact the developers (wouter.peters@wur.nl) to receive -updates of the code. See also: http://www.carbontracker.eu. - -This program is free software: you can redistribute it and/or modify it under the +"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters. +Users are recommended to contact the developers (wouter.peters@wur.nl) to receive +updates of the code. See also: http://www.carbontracker.eu. + +This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 3. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS @@ -14,18 +14,16 @@ program. If not, see <http://www.gnu.org/licenses/>.""" # obs.py """ -Author : aki +Author : peters Revision History: -File revised on Feb 2017. +File created on 28 Jul 2010. """ import os import sys import logging -#from da.baseclasses.statevector import filename import datetime as dtm -from string import strip from numpy import array, logical_and sys.path.append(os.getcwd()) @@ -86,8 +84,8 @@ class MethaneObservations(Observations): #evn = [s.tostring().lower() for s in evn] #evn = map(strip, evn) sites = ncf.get_variable('obs_id').take(subselect, axis=0) - sites = [s.tostring().lower() for s in sites] - sites = list(map(strip, sites)) + sites = [s.tostring().lower().decode("utf-8") for s in sites] + sites = list(map(str.strip, sites)) lats = ncf.get_variable('latitude').take(subselect, axis=0) lons = ncf.get_variable('longitude').take(subselect, axis=0) alts = ncf.get_variable('altitude').take(subselect, axis=0) @@ -107,7 +105,6 @@ class MethaneObservations(Observations): for n in range(len(dates)): obs[n] = obs[n] - #self.datalist.append(MoleFractionSample(ids[n], dates[n], sites[n], obs[n], 0.0, 0.0, 0.0, 0.0, flags[n], alts[n], lats[n], lons[n], evn[n], species[n], strategy[n], 0.0, self.obs_filename)) self.datalist.append( MoleFractionSample(ids[n], dates[n], sites[n], obs[n], 0.0, 0.0, 0.0, 0.0, 0.0, alts[n], lats[n], lons[n], '0000', 'ch4', 1.0, 0.0, self.obs_filename) ) logging.debug("Added %d observations to the Data list" % len(dates)) @@ -125,8 +122,7 @@ class MethaneObservations(Observations): ncf = io.ct_read(filename, method='read') ids = ncf.get_variable('obs_num') simulated = ncf.get_variable('flask') - #for i in xrange(simulated.shape[0]): - # print simulated[i,:] + ncf.close() logging.info("Successfully read data from model sample file (%s)" % filename) @@ -157,7 +153,7 @@ class MethaneObservations(Observations): """ f = io.CT_CDF(obsinputfile, method='create') - logging.debug('Creating new observations file for ObservationOperator (%s)' % obsinputfile) + logging.info('Creating new observations file for ObservationOperator (%s)' % obsinputfile) dimid = f.add_dim('obs', len(self.datalist)) dim200char = f.add_dim('string_of200chars', 200) @@ -168,7 +164,7 @@ class MethaneObservations(Observations): #return obsinputfile data = self.getvalues('id') - + savedict = io.std_savedict.copy() savedict['name'] = "obs_num" savedict['dtype'] = "int" @@ -284,7 +280,7 @@ class MethaneObservations(Observations): logging.debug('Model-data mismatch active sites : %d ' % self.n_sites_active) logging.debug('Model-data mismatch moved sites : %d ' % self.n_sites_moved) - cats = [k for k in list(sites_weights.keys()) if 'site.category' in k] + cats = [k for k in sites_weights.keys() if 'site.category' in k] SiteCategories = {} for key in cats: @@ -296,7 +292,7 @@ class MethaneObservations(Observations): SiteCategories[name] = {'error':error, 'may_localize':may_localize, 'may_reject':may_reject} #print name,SiteCategories[name] - active = [k for k in list(sites_weights.keys()) if 'site.active' in k] + active = [k for k in sites_weights.keys() if 'site.active' in k] site_info = {} for key in active: @@ -308,7 +304,7 @@ class MethaneObservations(Observations): for obs in self.datalist: obs.mdm = 1000.0 # default is very high model-data-mismatch, until explicitly set by script - if obs.code in site_info: + if obs.code in site_info: logging.debug("Observation found (%s)" % obs.code) obs.mdm = site_info[obs.code]['error'] * self.global_R_scaling obs.may_localize = site_info[obs.code]['may_localize'] diff --git a/da/methane/statevector.py b/da/methane/statevector.py index b75dafced7ca2387fb78022c19d7866c11e4a408..df0b0b652b724670c106c6356087811397d11979 100755 --- a/da/methane/statevector.py +++ b/da/methane/statevector.py @@ -1,8 +1,8 @@ -"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters. -Users are recommended to contact the developers (wouter.peters@wur.nl) to receive -updates of the code. See also: http://www.carbontracker.eu. - -This program is free software: you can redistribute it and/or modify it under the +"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters. +Users are recommended to contact the developers (wouter.peters@wur.nl) to receive +updates of the code. See also: http://www.carbontracker.eu. + +This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 3. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS @@ -43,7 +43,7 @@ class MethaneStateVector(StateVector): self.speciesdict = {'ch4': np.ones(self.nparams)} logging.debug("A species mask was created, only the following species are recognized in this system:") - for k in list(self.speciesdict.keys()): + for k in self.speciesdict.keys(): logging.debug(" -> %s" % k) def get_covariance(self, date, dacycle): diff --git a/da/methane/statevector_2lambdas.py b/da/methane/statevector_2lambdas.py index 6582cdd9dfdc6f00d37c9d37414f28bbc0b004c2..2c467996e6f439c1dfa93389ff95ea51470f223c 100755 --- a/da/methane/statevector_2lambdas.py +++ b/da/methane/statevector_2lambdas.py @@ -1,8 +1,8 @@ -"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters. -Users are recommended to contact the developers (wouter.peters@wur.nl) to receive -updates of the code. See also: http://www.carbontracker.eu. - -This program is free software: you can redistribute it and/or modify it under the +"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters. +Users are recommended to contact the developers (wouter.peters@wur.nl) to receive +updates of the code. See also: http://www.carbontracker.eu. + +This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 3. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS @@ -61,7 +61,7 @@ class MethaneStateVector(StateVector): # These list objects hold the data for each time step of lag in the system. Note that the ensembles for each time step consist # of lists of EnsembleMember objects, we define member 0 as the mean of the distribution and n=1,...,nmembers as the spread. - self.ensemble_members = list(range(self.nlag)) + self.ensemble_members = range(self.nlag) for n in range(self.nlag): self.ensemble_members[n] = [] @@ -119,7 +119,7 @@ class MethaneStateVector(StateVector): self.speciesdict = {'ch4': np.ones(self.nparams)} logging.debug("A species mask was created, only the following species are recognized in this system:") - for k in list(self.speciesdict.keys()): + for k in self.speciesdict.keys(): logging.debug(" -> %s" % k) def get_covariance(self, date, dacycle): @@ -135,7 +135,7 @@ class MethaneStateVector(StateVector): #----Arbitray covariance matrix ----# #fullcov = np.zeros((self.nparams, self.nparams), float) - #for i in xrange(self.nparams): + #for i in range(self.nparams): # #fullcov[i,i] = 1. # Identity matrix # fullcov[i,i] = 0.08 #fullcov[self.nparams-1,self.nparams-1] = 1.e-10