No commit message

No commit message

da/baseclasses/statevector.py

@@ -179,7 +179,7 @@ class StateVector(object):
         nparams = self.gridmap.max()
         self.griddict = {}
         for r in range(1, int(nparams) + 1):
-            sel = (self.gridmap.flat == r).nonzero()
+            sel = np.nonzero(self.gridmap.flat == r)
             if len(sel[0]) > 0: 
                 self.griddict[r] = sel
 
@@ -190,7 +190,7 @@ class StateVector(object):
         self.tcmatrix = np.zeros((self.nparams, 23), 'float') 
 
         for r in range(1, self.nparams + 1):
-            sel = (self.gridmap.flat == r).nonzero()
+            sel = np.nonzero(self.gridmap.flat == r)
             if len(sel[0]) < 1: 
                 continue
             else:

da/rc/carbontracker_random.rc

@@ -13,29 +13,32 @@
 
 !!! Info for the CarbonTracker data assimilation system
 
-datadir         : /Storage/CO2/carbontracker/input/ctdas_2012/ 
+datadir         : /store/empa/em05/parsenov/
 
 ! For ObsPack
-obspack.input.id   : obspack_co2_1_GLOBALVIEWplus_v2.1_2016-09-02
-obspack.input.dir  : ${datadir}/obspacks/${obspack.input.id}
-obs.sites.rc       : ${obspack.input.dir}/summary/sites_weights_Dec2016.rc
+obspack.input.id   : obspack
+obspack.input.dir  : ${datadir}
+!/obspack !/${obspack.input.id}
+obs.sites.rc       : ${obspack.input.dir}/summary/sites_weights_ctdas.rc
 
 ! Using a second ObsPack (from 1 January 2016)
-obspack.input.id2  : obspack_co2_1_NRT_v3.0_2016-06-06
-obspack.input.dir2 : ${datadir}/obspacks/${obspack.input.id2}
-obs.sites.rc2      : ${obspack.input.dir2}/summary/sites_weights_Dec2016.rc
-
-ocn.covariance  : ${datadir}/oceans/oif/cov_ocean.2000.01.nc 
-deltaco2.prefix : oif_p3_era40.dpco2
-
-bio.cov.dir     : ${datadir}/covariances/gridded_NH/
-bio.cov.prefix  : cov_ecoregion
-
-regtype         : gridded_oif30
-nparameters     : 9835 
-random.seed     : 4385
-regionsfile     : ${datadir}/covariances/gridded_NH/griddedNHparameters.nc
-random.seed.init: ${datadir}/randomseedinit.pickle
+!obspack.input.id2  : obspack_co2_1_NRT_v3.0_2016-06-06
+!obspack.input.dir2 : ${datadir}/obspacks/${obspack.input.id2}
+!obs.sites.rc2      : ${obspack.input.dir2}/summary/sites_weights_Dec2016.rc
+
+!ocn.covariance  : ${datadir}/oceans/oif/cov_ocean.2000.01.nc 
+!deltaco2.prefix : oif_p3_era40.dpco2
+
+!bio.cov.dir     : ${datadir}/covariances/gridded_NH/
+!bio.cov.prefix  : cov_ecoregion
+
+!regtype         : gridded_oif30
+nparameters     : 11 
+!random.seed     : 4385
+regionsfile     : /store/empa/em05/parsenov/CTDAS/ctdas-py3/da/analysis/regions_ok.nc
+extendedregionsfile     : /store/empa/em05/parsenov/CTDAS/ctdas-py3/da/analysis/regions_ok.nc
+!regionsfile     : ${datadir}/covariances/gridded_NH/griddedNHparameters.nc
+!random.seed.init: ${datadir}/randomseedinit.pickle
 
 ! Include a naming scheme for the variables
 

da/tools/initexit.py

@@ -332,7 +332,7 @@ class CycleControl(dict):
 
             strippedname = os.path.split(self['jobrcfilename'])[-1]
             self['jobrcfilename'] = os.path.join(self['dir.exec'], strippedname)
-            shutil.copy(os.path.join(self.dasystem['regionsfile']),os.path.join(self['dir.exec'],'da','analysis','copied_regions.nc'))
+#            shutil.copy(os.path.join(self.dasystem['regionsfile']),os.path.join(self['dir.exec'],'da','analysis','copied_regions.nc'))
             logging.info('Copied regions file to the analysis directory: %s'%os.path.join(self.dasystem['regionsfile'])) 
             if self.dasystem.has_key('extendedregionsfile'):
                 shutil.copy(os.path.join(self.dasystem['extendedregionsfile']),os.path.join(self['dir.exec'],'da','analysis','copied_regions_extended.nc')) 

template.jb

@@ -8,7 +8,7 @@
 echo "All output piped to file template.log"
 source /usr/local/Modules/3.2.8/init/sh
 source /opt/intel/bin/ifortvars.sh intel64
-export HOST='capegrim'
+export HOST='daint'
 module load python
 export icycle_in_job=999
 python template.py rc=template.rc -v $1 >& template.log &

template.py

@@ -25,17 +25,16 @@ sys.path.append(os.getcwd())
 # Next, import the tools needed to initialize a data assimilation cycle
 #################################################################################################
 
-from da.tools.initexit import start_logger, validate_opts_args, parse_options, CycleControl 
-from da.tools.pipeline import ensemble_smoother_pipeline, header, footer, analysis_pipeline, archive_pipeline
-from da.platform.cartesius import CartesiusPlatform 
-from da.carbondioxide.dasystem import CO2DaSystem 
-from da.carbondioxide.optimizer import CO2Optimizer
-from da.carbondioxide.obspack_geocarbon import ObsPackObservations
-#from da.carbondioxide.statevector import CO2StateVector 
-from da.co2gridded.statevector import CO2GriddedStateVector 
-#from da.carbondioxide.obspack import ObsPackObservations 
-#from da.carbondioxide.obs import CO2Observations 
-from da.tm5.observationoperator import TM5ObservationOperator 
+from da.tools.initexit import start_logger, validate_opts_args, parse_options, CycleControl
+from da.tools.pipeline import ensemble_smoother_pipeline, header, footer
+from da.platform.maunaloa import MaunaloaPlatform 
+from da.baseclasses.dasystem import DaSystem 
+from da.baseclasses.statevector import StateVector 
+from da.carbondioxide.obspack_globalviewplus2 import ObsPackObservations 
+from da.baseclasses.optimizer import Optimizer
+from da.baseclasses.observationoperator import ObservationOperator
+#from da.analysis.expand_fluxes import save_weekly_avg_1x1_data, save_weekly_avg_state_data, save_weekly_avg_tc_data, save_weekly_avg_ext_tc_data
+#from da.analysis.expand_molefractions import write_mole_fractions
 
 
 #################################################################################################
@@ -43,9 +42,6 @@ from da.tm5.observationoperator import TM5ObservationOperator
 #################################################################################################
 
 start_logger()
-logging.info(header + "CTDAS Copyright (C) 2017, Wouter Peters " + footer)
-logging.info(header + "This program comes with ABSOLUTELY NO WARRANTY." + footer)
-logging.info(header + "This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 3." + footer)
 opts, args = parse_options()
 opts, args = validate_opts_args(opts, args)
 
@@ -55,26 +51,19 @@ opts, args = validate_opts_args(opts, args)
 
 dacycle = CycleControl(opts, args)
 
-###########################################################################################
-### IMPORT THE APPLICATION SPECIFIC MODULES HERE, TO BE PASSED INTO THE MAIN PIPELINE!!! ##
-###########################################################################################
 
-
-platform = CartesiusPlatform()
-dasystem = CO2DaSystem(dacycle['da.system.rc'])
-obsoperator = TM5ObservationOperator(dacycle['da.obsoperator.rc'])
-samples = ObsPackObservations()
-#samples     = CtObservations()
-statevector = CO2GriddedStateVector()
-#statevector = CO2StateVector()
-optimizer = CO2Optimizer()
+platform = MaunaloaPlatform()
+dasystem = DaSystem(dacycle['da.system.rc'])
+obsoperator = ObservationOperator(dacycle['da.obsoperator.rc'])
+samples     = ObsPackObservations()
+statevector = StateVector()
+optimizer = Optimizer()
 
 ##########################################################################################
 ################### ENTER THE PIPELINE WITH THE OBJECTS PASSED BY THE USER ###############
 ##########################################################################################
 
 
-
 logging.info(header + "Entering Pipeline " + footer) 
 
 ensemble_smoother_pipeline(dacycle, platform, dasystem, samples, statevector, obsoperator, optimizer)
@@ -84,9 +73,14 @@ ensemble_smoother_pipeline(dacycle, platform, dasystem, samples, statevector, ob
 ################### All done, extra stuff can be added next, such as analysis
 ##########################################################################################
 
-analysis_pipeline(dacycle, platform, dasystem, samples, statevector )
+logging.info(header + "Starting analysis" + footer) 
+sys.exit(0)
 
-archive_pipeline(dacycle, platform, dasystem)
+save_weekly_avg_1x1_data(dacycle, statevector)
+save_weekly_avg_state_data(dacycle, statevector)
+save_weekly_avg_tc_data(dacycle, statevector)
+save_weekly_avg_ext_tc_data(dacycle)
+write_mole_fractions(dacycle)
 
 sys.exit(0)
 

template.rc

@@ -28,8 +28,8 @@
 !
 ! The time for which to start and end the data assimilation experiment in format YYYY-MM-DD HH:MM:SS
 
-time.start          : 2000-01-01 00:00:00
-time.finish         : 2000-01-15 00:00:00
+time.start          : 2012-01-01 00:00:00
+time.finish         : 2012-01-15 00:00:00
 
 ! Whether to restart the CTDAS system from a previous cycle, or to start the sequence fresh. Valid entries are T/F/True/False/TRUE/FALSE
 
@@ -37,29 +37,29 @@ time.restart        : False
 
 ! The length of a cycle is given in days, such that the integer 7 denotes the typically used weekly cycle. Valid entries are integers > 1
 
-time.cycle          : 7
+time.cycle          : 1
 
 ! The number of cycles of lag to use for a smoother version of CTDAS. CarbonTracker CO2 typically uses 5 weeks of lag. Valid entries are integers > 0
 
-time.nlag           : 5
+time.nlag           : 3
 
 ! The directory under which the code, input, and output will be stored. This is the base directory for a run. The word
 ! '/' will be replaced through the start_ctdas.sh script by a user-specified folder name. DO NOT REPLACE
 
-dir.da_run          : template
+dir.da_run          : /scratch/snx3000/parsenov/projname
 
 ! The resources used to complete the data assimilation experiment. This depends on your computing platform.
 ! The number of cycles per job denotes how many cycles should be completed before starting a new process or job, this
 ! allows you to complete many cycles before resubmitting a job to the queue and having to wait again for resources.
 ! Valid entries are integers > 0
 
-da.resources.ncycles_per_job : 2
+da.resources.ncycles_per_job : 1
 
 ! The ntasks specifies the number of threads to use for the MPI part of the code, if relevant. Note that the CTDAS code
 ! itself is not parallelized and the python code underlying CTDAS does not use multiple processors. The chosen observation
 ! operator though might use many processors, like TM5. Valid entries are integers > 0
 
-da.resources.ntasks : 30
+da.resources.ntasks : 1
 
 ! This specifies the amount of wall-clock time to request for each job. Its value depends on your computing platform and might take
 ! any form appropriate for your system. Typically, HPC queueing systems allow you a certain number of hours of usage before 
@@ -77,11 +77,11 @@ da.system           : CarbonTracker
 
 ! The specific settings for your system are read from a separate rc-file, which points to the data directories, observations, etc
 
-da.system.rc        : da/rc/carbontracker.rc
+da.system.rc        : da/rc/carbontracker_random.rc
 
 ! This flag should probably be moved to the da.system.rc file. It denotes which type of filtering to use in the optimizer
 
-da.system.localization : CT2007
+da.system.localization : None
 
 ! Info on the observation operator to be used, these keys help to identify the settings for the transport model in this case
 
@@ -99,7 +99,7 @@ da.obsoperator.rc      : ${da.obsoperator.home}/tm5-ctdas-ei-zoom.rc
 ! The number of ensemble members used in the experiment. Valid entries are integers > 2
 !
 
-da.optimizer.nmembers  : 150
+da.optimizer.nmembers  : 10
 
 ! Finally, info on the archive task, if any. Archive tasks are run after each cycle to ensure that the results of each cycle are
 ! preserved, even if you run on scratch space or a temporary disk. Since an experiment can take multiple weeks to complete, moving
@@ -121,7 +121,7 @@ da.optimizer.nmembers  : 150
 
 !task.rsync.alldata.sourcedirs : ${dir.da_run}
 !task.rsync.alldata.destinationdir : you@yourserver.com:/yourfolder/
-!task.rsync.alldata.flags : -auv -e ssh
+!task.rsync.alldata.flags g -auv -e ssh
 
 ! Repeated for rsync task 2, note that we only back up the analysis and output dirs here