Merge branch 'master' into 'master'

We changed the master project to once again run the random observation...

See merge request ctdas/CTDAS!12

da/co2gridded/dasystem.py

@@ -56,7 +56,7 @@ class CO2GriddedDaSystem(DaSystem):
                            'regtype']
 
 
-        for k, v in self.iteritems():
+        for k, v in self.items():
             if v == 'True' : self[k] = True
             if v == 'False': self[k] = False
 

da/co2gridded/statevector.py

@@ -145,7 +145,7 @@ class CO2GriddedStateVector(StateVector):
             dof += np.sum(s) ** 2 / sum(s ** 2)
             try:
                 C = np.linalg.cholesky(matrix)
-            except np.linalg.linalg.LinAlgError, err:
+            except np.linalg.linalg.LinAlgError as err:
                 logging.error('Cholesky decomposition has failed ')
                 logging.error('For a matrix of dimensions: %d' % matrix.shape[0])
                 logging.debug(err)

da/co2gridded/statevectorNHgridded.py

@@ -89,7 +89,7 @@ class CO2GriddedStateVector(StateVector):
 
             if 'pco2' in file or 'cov_ocean' in file: 
                 cov_ocn = f.get_variable('CORMAT')
-                parnr = range(9805,9835)
+                parnr = list(range(9805,9835))
                 cov = cov_ocn
             elif 'tropic' in file:
                 cov = f.get_variable('covariance')
@@ -157,7 +157,7 @@ class CO2GriddedStateVector(StateVector):
             dof += np.sum(s) ** 2 / sum(s ** 2)
             try:
                 C = np.linalg.cholesky(matrix)
-            except np.linalg.linalg.LinAlgError, err:
+            except np.linalg.linalg.LinAlgError as err:
                 logging.error('Cholesky decomposition has failed ')
                 logging.error('For a matrix of dimensions: %d' % matrix.shape[0])
                 logging.debug(err)

da/rc/carbontracker_random.rc

@@ -13,31 +13,28 @@
 
 !!! Info for the CarbonTracker data assimilation system
 
-datadir         : /store/empa/em05/parsenov/
+datadir         : /Storage/CO2/carbontracker/input/ctdas_2016/
 
 ! For ObsPack
-obspack.input.id   : obspack
-obspack.input.dir  : ${datadir}
-!/obspack !/${obspack.input.id}
-obs.sites.rc       : ${obspack.input.dir}/summary/sites_weights_ctdas.rc
+obspack.input.id   : obspack_co2_1_GLOBALVIEWplus_v2.1_2016-09-02
+obspack.input.dir  : ${datadir}/obspacks/${obspack.input.id}
+obs.sites.rc       : ${obspack.input.dir}/summary/sites_weights_Dec2016.rc
 
 ! Using a second ObsPack (from 1 January 2016)
 !obspack.input.id2  : obspack_co2_1_NRT_v3.0_2016-06-06
 !obspack.input.dir2 : ${datadir}/obspacks/${obspack.input.id2}
 !obs.sites.rc2      : ${obspack.input.dir2}/summary/sites_weights_Dec2016.rc
 
-!ocn.covariance  : ${datadir}/oceans/oif/cov_ocean.2000.01.nc 
-!deltaco2.prefix : oif_p3_era40.dpco2
+ocn.covariance  : ${datadir}/oceans/oif/cov_ocean.2000.01.nc 
+deltaco2.prefix : oif_p3_era40.dpco2
 
-!bio.cov.dir     : ${datadir}/covariances/gridded_NH/
-!bio.cov.prefix  : cov_ecoregion
+bio.cov.dir     : ${datadir}/covariances/gridded_NH/
+bio.cov.prefix  : cov_ecoregion
 
-!regtype         : gridded_oif30
-nparameters     : 11 
-!random.seed     : 4385
-regionsfile     : /store/empa/em05/parsenov/CTDAS/ctdas-py3/da/analysis/regions_ok.nc
-!extendedregionsfile     : /store/empa/em05/parsenov/CTDAS/ctdas-py3/da/analysis/regions_ok.nc
-!regionsfile     : ${datadir}/covariances/gridded_NH/griddedNHparameters.nc
+regtype         : gridded_oif30
+nparameters     : 9835 
+random.seed     : 4385
+regionsfile     : ${datadir}/covariances/gridded_NH/griddedNHparameters.nc
 !random.seed.init: ${datadir}/randomseedinit.pickle
 
 ! Include a naming scheme for the variables

templates/template.jb

@@ -5,10 +5,8 @@
 #$ 06:30:00
 #$ /bin/sh
 
-echo "All output piped to file template.log"
-source /usr/local/Modules/3.2.8/init/sh
-source /opt/intel/bin/ifortvars.sh intel64
-export HOST='daint'
-module load python
-export icycle_in_job=999
+echo "All output piped to file ct_base_test.log"
+export HOST='capegrim'
+export icycle_in_job=1
+python --version
 python template.py rc=template.rc -v $1 >& template.log &

templates/template.py

@@ -29,12 +29,11 @@ from da.tools.initexit import start_logger, validate_opts_args, parse_options, C
 from da.tools.pipeline import ensemble_smoother_pipeline, header, footer
 from da.platform.capegrim import CapeGrimPlatform
 from da.baseclasses.dasystem import DaSystem
-from da.baseclasses.statevector import StateVector 
+from da.co2gridded.statevector import CO2GriddedStateVector 
 from da.carbondioxide.obspack_globalviewplus2 import ObsPackObservations
-from da.carbondioxide.optimizer import CO2Optimizer
-from da.baseclasses.observationoperator import ObservationOperator
-#from da.analysis.expand_fluxes import save_weekly_avg_1x1_data, save_weekly_avg_state_data, save_weekly_avg_tc_data, save_weekly_avg_ext_tc_data
-#from da.analysis.expand_molefractions import write_mole_fractions
+from da.baseclasses.optimizer import Optimizer
+from da.baseclasses.observationoperator import RandomizerObservationOperator
+
 
 
 #################################################################################################
@@ -51,13 +50,12 @@ opts, args = validate_opts_args(opts, args)
 
 dacycle = CycleControl(opts, args)
 
-
-platform = MaunaloaPlatform()
+platform = CapeGrimPlatform()
 dasystem = DaSystem(dacycle['da.system.rc'])
-obsoperator = ObservationOperator(dacycle['da.obsoperator.rc'])
+obsoperator = RandomizerObservationOperator(dacycle['da.obsoperator.rc'])
 samples     = ObsPackObservations()
-statevector = StateVector()
-optimizer = CO2Optimizer()
+statevector = CO2GriddedStateVector()
+optimizer = Optimizer()
 
 ##########################################################################################
 ################### ENTER THE PIPELINE WITH THE OBJECTS PASSED BY THE USER ###############

templates/template.rc

@@ -28,8 +28,8 @@
 !
 ! The time for which to start and end the data assimilation experiment in format YYYY-MM-DD HH:MM:SS
 
-time.start          : 2016-01-01 00:00:00
-time.finish         : 2016-02-01 00:00:00
+time.start          : 2012-01-01 00:00:00
+time.finish         : 2012-02-01 00:00:00
 
 ! Whether to restart the CTDAS system from a previous cycle, or to start the sequence fresh. Valid entries are T/F/True/False/TRUE/FALSE
 
@@ -37,16 +37,16 @@ time.restart        : False
 
 ! The length of a cycle is given in days, such that the integer 7 denotes the typically used weekly cycle. Valid entries are integers > 1
 
-time.cycle          : 7
+time.cycle          : 1
 
 ! The number of cycles of lag to use for a smoother version of CTDAS. CarbonTracker CO2 typically uses 5 weeks of lag. Valid entries are integers > 0
 
-time.nlag           : 3
+time.nlag           : 2
 
 ! The directory under which the code, input, and output will be stored. This is the base directory for a run. The word
 ! '/' will be replaced through the start_ctdas.sh script by a user-specified folder name. DO NOT REPLACE
 
-dir.da_run          : /scratch/snx3000/parsenov/projname
+dir.da_run          : template
 
 ! The resources used to complete the data assimilation experiment. This depends on your computing platform.
 ! The number of cycles per job denotes how many cycles should be completed before starting a new process or job, this