We changed the master project to once again run the random observation...

We changed the master project to once again run the random observation operator in python 3, using the gridded statevector and the ObsPack GlobalViewplus2 on capegrim. Next to do is to make this project also run without the specific parts but instead with baseclasses, and to program something that allows different templates on different machines.

da/co2gridded/dasystem.py

@@ -56,7 +56,7 @@ class CO2GriddedDaSystem(DaSystem):
                            'regtype']
 
 
-        for k, v in self.iteritems():
+        for k, v in self.items():
             if v == 'True' : self[k] = True
             if v == 'False': self[k] = False
 

da/co2gridded/statevector.py

@@ -145,7 +145,7 @@ class CO2GriddedStateVector(StateVector):
             dof += np.sum(s) ** 2 / sum(s ** 2)
             try:
                 C = np.linalg.cholesky(matrix)
-            except np.linalg.linalg.LinAlgError, err:
+            except np.linalg.linalg.LinAlgError as err:
                 logging.error('Cholesky decomposition has failed ')
                 logging.error('For a matrix of dimensions: %d' % matrix.shape[0])
                 logging.debug(err)

da/co2gridded/statevectorNHgridded.py

@@ -89,7 +89,7 @@ class CO2GriddedStateVector(StateVector):
 
             if 'pco2' in file or 'cov_ocean' in file: 
                 cov_ocn = f.get_variable('CORMAT')
-                parnr = range(9805,9835)
+                parnr = list(range(9805,9835))
                 cov = cov_ocn
             elif 'tropic' in file:
                 cov = f.get_variable('covariance')
@@ -157,7 +157,7 @@ class CO2GriddedStateVector(StateVector):
             dof += np.sum(s) ** 2 / sum(s ** 2)
             try:
                 C = np.linalg.cholesky(matrix)
-            except np.linalg.linalg.LinAlgError, err:
+            except np.linalg.linalg.LinAlgError as err:
                 logging.error('Cholesky decomposition has failed ')
                 logging.error('For a matrix of dimensions: %d' % matrix.shape[0])
                 logging.debug(err)

da/rc/carbontracker_random.rc

@@ -13,31 +13,28 @@
 
 !!! Info for the CarbonTracker data assimilation system
 
-datadir         : /store/empa/em05/parsenov/
+datadir         : /Storage/CO2/carbontracker/input/ctdas_2016/
 
 ! For ObsPack
-obspack.input.id   : obspack
-obspack.input.dir  : ${datadir}
-!/obspack !/${obspack.input.id}
-obs.sites.rc       : ${obspack.input.dir}/summary/sites_weights_ctdas.rc
+obspack.input.id   : obspack_co2_1_GLOBALVIEWplus_v2.1_2016-09-02
+obspack.input.dir  : ${datadir}/obspacks/${obspack.input.id}
+obs.sites.rc       : ${obspack.input.dir}/summary/sites_weights_Dec2016.rc
 
 ! Using a second ObsPack (from 1 January 2016)
 !obspack.input.id2  : obspack_co2_1_NRT_v3.0_2016-06-06
 !obspack.input.dir2 : ${datadir}/obspacks/${obspack.input.id2}
 !obs.sites.rc2      : ${obspack.input.dir2}/summary/sites_weights_Dec2016.rc
 
-!ocn.covariance  : ${datadir}/oceans/oif/cov_ocean.2000.01.nc 
-!deltaco2.prefix : oif_p3_era40.dpco2
+ocn.covariance  : ${datadir}/oceans/oif/cov_ocean.2000.01.nc 
+deltaco2.prefix : oif_p3_era40.dpco2
 
-!bio.cov.dir     : ${datadir}/covariances/gridded_NH/
-!bio.cov.prefix  : cov_ecoregion
+bio.cov.dir     : ${datadir}/covariances/gridded_NH/
+bio.cov.prefix  : cov_ecoregion
 
-!regtype         : gridded_oif30
-nparameters     : 11 
-!random.seed     : 4385
-regionsfile     : /store/empa/em05/parsenov/CTDAS/ctdas-py3/da/analysis/regions_ok.nc
-!extendedregionsfile     : /store/empa/em05/parsenov/CTDAS/ctdas-py3/da/analysis/regions_ok.nc
-!regionsfile     : ${datadir}/covariances/gridded_NH/griddedNHparameters.nc
+regtype         : gridded_oif30
+nparameters     : 9835 
+random.seed     : 4385
+regionsfile     : ${datadir}/covariances/gridded_NH/griddedNHparameters.nc
 !random.seed.init: ${datadir}/randomseedinit.pickle
 
 ! Include a naming scheme for the variables

templates/template.jb

@@ -5,10 +5,8 @@
 #$ 06:30:00
 #$ /bin/sh
 
-echo "All output piped to file template.log"
-source /usr/local/Modules/3.2.8/init/sh
-source /opt/intel/bin/ifortvars.sh intel64
-export HOST='daint'
-module load python
-export icycle_in_job=999
+echo "All output piped to file ct_base_test.log"
+export HOST='capegrim'
+export icycle_in_job=1
+python --version
 python template.py rc=template.rc -v $1 >& template.log &

templates/template.py

@@ -27,14 +27,13 @@ sys.path.append(os.getcwd())
 
 from da.tools.initexit import start_logger, validate_opts_args, parse_options, CycleControl
 from da.tools.pipeline import ensemble_smoother_pipeline, header, footer
-from da.platform.capegrim import CapeGrimPlatform 
-from da.baseclasses.dasystem import DaSystem 
-from da.baseclasses.statevector import StateVector 
-from da.carbondioxide.obspack_globalviewplus2 import ObsPackObservations 
-from da.carbondioxide.optimizer import CO2Optimizer
-from da.baseclasses.observationoperator import ObservationOperator
-#from da.analysis.expand_fluxes import save_weekly_avg_1x1_data, save_weekly_avg_state_data, save_weekly_avg_tc_data, save_weekly_avg_ext_tc_data
-#from da.analysis.expand_molefractions import write_mole_fractions
+from da.platform.capegrim import CapeGrimPlatform
+from da.baseclasses.dasystem import DaSystem
+from da.co2gridded.statevector import CO2GriddedStateVector 
+from da.carbondioxide.obspack_globalviewplus2 import ObsPackObservations
+from da.baseclasses.optimizer import Optimizer
+from da.baseclasses.observationoperator import RandomizerObservationOperator
+
 
 
 #################################################################################################
@@ -51,13 +50,12 @@ opts, args = validate_opts_args(opts, args)
 
 dacycle = CycleControl(opts, args)
 
-
-platform = MaunaloaPlatform()
+platform = CapeGrimPlatform()
 dasystem = DaSystem(dacycle['da.system.rc'])
-obsoperator = ObservationOperator(dacycle['da.obsoperator.rc'])
+obsoperator = RandomizerObservationOperator(dacycle['da.obsoperator.rc'])
 samples     = ObsPackObservations()
-statevector = StateVector()
-optimizer = CO2Optimizer()
+statevector = CO2GriddedStateVector()
+optimizer = Optimizer()
 
 ##########################################################################################
 ################### ENTER THE PIPELINE WITH THE OBJECTS PASSED BY THE USER ###############

templates/template.rc

@@ -28,8 +28,8 @@
 !
 ! The time for which to start and end the data assimilation experiment in format YYYY-MM-DD HH:MM:SS
 
-time.start          : 2016-01-01 00:00:00
-time.finish         : 2016-02-01 00:00:00
+time.start          : 2012-01-01 00:00:00
+time.finish         : 2012-02-01 00:00:00
 
 ! Whether to restart the CTDAS system from a previous cycle, or to start the sequence fresh. Valid entries are T/F/True/False/TRUE/FALSE
 
@@ -37,16 +37,16 @@ time.restart        : False
 
 ! The length of a cycle is given in days, such that the integer 7 denotes the typically used weekly cycle. Valid entries are integers > 1
 
-time.cycle          : 7
+time.cycle          : 1
 
 ! The number of cycles of lag to use for a smoother version of CTDAS. CarbonTracker CO2 typically uses 5 weeks of lag. Valid entries are integers > 0
 
-time.nlag           : 3
+time.nlag           : 2
 
 ! The directory under which the code, input, and output will be stored. This is the base directory for a run. The word
 ! '/' will be replaced through the start_ctdas.sh script by a user-specified folder name. DO NOT REPLACE
 
-dir.da_run          : /scratch/snx3000/parsenov/projname
+dir.da_run          : template
 
 ! The resources used to complete the data assimilation experiment. This depends on your computing platform.
 ! The number of cycles per job denotes how many cycles should be completed before starting a new process or job, this