updated the way rc-files are used in das.py and in dasjet.py, new name for forecast.nmembers

da/baseclasses/dasystem.py

@@ -10,22 +10,22 @@ File created on 26 Aug 2010.
 
 The DaSystem class is found in the module :mod:`dasystem`, or in a specific implementation under the da/ source tree. It is derived from the standard python :class:`dictionary` object. 
 
-It describes the details of the data assimilation system used (i.e., CarbonTracker, or CT Methane, or ....).
-
-An example rc-file for CarbonTracker is:::
-
-    !!! Info for the CarbonTracker data assimilation system
-
-    datadir         : /data/CO2/carbontracker/ct08/
-    obs.input.dir   : ${datadir}/obsnc/with_fillvalue/
-    obs.input.fname : obs_final.nc
-    ocn.covariance  : ${datadir}/oif_p3_era40.dpco2.2000.01.hdf 
-    bio.covariance  : ${datadir}/covariance_bio_olson19.nc
-    deltaco2.prefix : oif_p3_era40.dpco2
-    regtype         : olson19_oif30
-    nparameters     : 240
-    random.seed     : 4385
-    regionsfile     : transcom_olson19_oif30.hdf
+It describes the details of the data assimilation system used (i.e., CarbonTracker, or CT Methane, or ....) ::
+
+    datadir         : /Volumes/Storage/CO2/carbontracker/input/ct08/   ! The directory where input data is found
+    obs.input.dir   : ${datadir}/obsnc/with_fillvalue                  ! the observation input dir
+    obs.input.fname : obs_forecast.nc                                  ! the observation input file
+    ocn.covariance  : ${datadir}/oif_p3_era40.dpco2.2000.01.hdf        ! the ocean flux covariance file
+    bio.covariance  : ${datadir}/covariance_bio_olson19.nc             ! the biosphere flux covariance file
+    deltaco2.prefix : oif_p3_era40.dpco2                               ! the type of ocean product used
+    regtype         : olson19_oif30                                    ! the ecoregion definitions
+    nparameters     : 240                                              ! the number of parameters to solve for
+    random.seed     : 4385                                             ! the random seed for the first cycle
+    regionsfile     : transcom_olson19_oif30.hdf                       ! the ecoregion defintion mask file
+
+    ! Info on the sites file used
+
+    obs.sites.rc        : ${datadir}/sites_and_weights_co2.ct10.rc     ! the weights in the covariance matric of each obs
 
 The full baseclass description:
 

da/baseclasses/optimizer.py

@@ -458,7 +458,7 @@ if __name__ == "__main__":
 
     nobs      = len(samples.Data)
     dims      = ( int(DaCycle['time.nlag']),
-                  int(DaCycle['forecast.nmembers']),
+                  int(DaCycle['da.optimizer.nmembers']),
                   int(DaCycle.DaSystem['nparameters']),
                   nobs,  )
 

da/baseclasses/statevector.py

@@ -190,7 +190,7 @@ class StateVector(object):
         """
 
         dims                = ( int(self.DaCycle['time.nlag']),
-                            int(self.DaCycle['forecast.nmembers']),
+                            int(self.DaCycle['da.optimizer.nmembers']),
                             int(self.DaCycle.DaSystem['nparameters']),
                            )
 
@@ -423,7 +423,7 @@ if __name__ == "__main__":
     print DaCycle
 
     dims        = ( int(DaCycle['time.nlag']),
-                  int(DaCycle['forecast.nmembers']),
+                  int(DaCycle['da.optimizer.nmembers']),
                   int(DaCycle.DaSystem['nparameters']),
                   )
 

da/ct/optimizer.py

@@ -91,7 +91,7 @@ if __name__ == "__main__":
 
     nobs      = 100
     dims      = ( int(DaCycle['time.nlag']),
-                  int(DaCycle['forecast.nmembers']),
+                  int(DaCycle['da.optimizer.nmembers']),
                   int(DaCycle.DaSystem['nparameters']),
                   nobs,  )
 

da/doc/source/tut_chapter1.rst

@@ -5,11 +5,11 @@ Chapter 1: Configuring CTDAS to run an experiment
 
 **In which you will learn to modify the three primary files that together control a CTDAS experiment**
 
-Modify the two primary rc-files
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Step 1: Modify the two primary rc-files
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 When you checkout CTDAS from the subversion server, there are two example rc-files that you need to modify. 
-   1. The ``das.rc`` file, which describes your CTDAS configuration with respect to experiment name, time period, and lag
+   1. The ``da.rc`` file, which describes your CTDAS configuration with respect to experiment name, time period, and lag
    2. The ``carbontracker.rc`` file, which describes your CTDAS configuration with respect to observations and statevector
 
 .. note:: The example files are found in the da/rc/ directory of your CTDAS tree. You are encouraged to create copies of these 
@@ -28,14 +28,14 @@ can be replaced by: ::
     dir.da_run          : /scratch/${USER}/my_first_ctdas_run
 
 Which, as you have likely guessed, will change the location where CTDAS creates a directory structure and places input/output files 
-for your simulation. 
+for your simulation. See :mod:`initexit` for more information on these settings. It is especially important to set the keys ::
 
-Where the ``das.rc`` file is rather self-explanatory, the ``carbontracker.rc`` file has keys that refer to the inner workings of CTDAS,
-such as: ::
+    da.system.rc        : da/rc/carbontracker.rc                ! the settings needed in your inversion system
+    da.obsoperator.rc      : ${HOME}/Modeling/TM5/tm5-ctdas.rc  ! the rc-file needed to run youobservation operator
 
-    ocn.covariance  : ${datadir}/oif_p3_era40.dpco2.2000.01.hdf 
-    regionsfile     : transcom_olson19_oif30.hdf
-    obs.sites.rc    : ${datadir}/sites_and_weights_co2.ct10.rc
+correctly. The first one refers to the rc-file (2) described above, while the second one refers to the rc-file you used to compile the TM5 model in :ref:`Chapter 0 <tut_chapter0>`.
+
+Where the ``da.rc`` file is rather self-explanatory, the ``carbontracker.rc`` file has keys that refer to the inner workings of CTDAS as described in :mod:`dasystem`
 
 These specify files, or paths to files, that are needed to construct the :ref:`statevector <Statevector>` object and the 
 :ref:`observations <Observation>` object. For instance, they allow CTDAS to map parameters to the global grid, or to 
@@ -45,15 +45,15 @@ construct model-data-mismatches for the sites that will be used in the assimilat
          fact that they are missing.
 
 
-Modify the primary run script
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Step 2: Modify the python run script
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The primary run script for CTDAS is simply called ``das.py`` and can also be found in the ``da/examples/`` subdirectory. As before,
-you should copy this file before modyfing it.
+You are required to create your own python run script for CTDAS. In this tutorial, we assume it is simply called ``das.py`` and an example 
+is checked out in the ``da/tools/`` trunk directory. As before, you should copy this file before modyfing it.
 
 .. note:: The primary run script needs to be located in the main directory of the CTDAS tree, i.e., ``${yourdir}/da/ct/trunk/``
 
-The header of this script can contain settings that are needed to run it on your platform (e.g., queue flags), and is then followed by 
+The script first initializes some python objects needed to log activity and to parse command line arguments, and is then followed by 
 a block where all the modules that are needed in your experiment are imported. The example below shows the import of several classes that 
 are needed to run the CarbonTracker CO2 system on a computer referred to as MaunaLoa, using TM5 as 
 an :ref:`observationoperator <Observation Operator>` ::
@@ -69,28 +69,30 @@ an :ref:`observationoperator <Observation Operator>` ::
     from da.tm5.observationoperator import TM5ObservationOperator 
     from da.ct.optimizer import CtOptimizer
     
-Once the classes are loaded successfully, the objects can be created. Note how the 2nd and 3rd object make use of rc-files! ::
+Once the classes are loaded successfully, the objects are created. ::
 
     PlatForm    = MaunaloaPlatForm()
-    DaSystem    = CtDaSystem('carbontracker.rc')
-    ObsOperator = TM5ObservationOperator('/Users/peters/Modeling/TM5/tm5-ctdas.rc')
+    DaSystem    = CtDaSystem(DaCycle['da.system.rc'])
+    ObsOperator = TM5ObservationOperator(DaCycle['da.obsoperator.rc'])
     Samples     = CtObservations()
     StateVector = CtStateVector()
     Optimizer   = CtOptimizer()
 
+.. note:: See how the initilization of the DaSystem and ObservationOperator object make use of the keys specified in your primary rc-file ! 
+
 Modification of these objects might be desirable for more advanced users, and in case of the :ref:`platform <Platform>` object, even 
 necessary (see next section). Once the objects are created, they are simply passed to a pipeline for the CTDAS. In the first 
 chapter of the tutorial, we will assume this pipeline is immutable.
 
-Thus, once you have modified the headers of the ``das.py`` script, and changed the references to the rc-files, you can focus on the last 
-configuration step:
+In order to complete the preparations for your first run, you might have to also go through Step 3:
 
-Creating a PlatForm object for your system
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Step 3: Creating a PlatForm object for your system
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 From the description of the :ref:`platform <Platform>` object, you will understand that this object is (partly) unique for each user, 
 or at least for the computing environment of each user. Information on the computing system is therefore coded into a specific python object. 
-**This object will need to be created for your system by you.**
+
+.. warning:: **This object will need to be created for your system by you.**
 
 Luckily, part of the work is already done. In the ``da/baseclasses`` subdirectory you will find a baseclass :ref:`platform <Platform>` 
 which serves as a blueprint for your own ``Platform`` object. This is done through class inheritance. As an example, you can open one 
@@ -128,13 +130,14 @@ While the baseclass did not have any functionality, a call to the ``GetJobTempla
 a job script on NOAA's "jet" supercomputer, so that we can submit jobs to its queue. **By modifying each of the methods in your own 
 derived PlatForm class in the same way, you can make each method work on your system**. 
 
+.. note:: As default, you could first try if the simple baseclass PlatForm works for you
+
 Once you have created your own PlatForm object, and you have successfully imported and instantiated it in your primary python run script, 
 you are ready to give your first experiment a try.
 
+
 .. note:: Sometimes it is faster and easier to test your newly created class 'offline'. At the end of your module, following the __main__ 
          section you can add lines to test your PlatForm object before plugging it into the CTDAS. 
 
-.. note:: I am hoping to one day implement simple Test() functions for all methods so that code can be formally tested before 
-         using it in a real world application. 
 
 

da/doc/source/tut_chapter2.rst

-.. _tut_chapter1:
+.. _tut_chapter2:
 
 Chapter 2: Running your first experiment
 ----------------------------------------------------

da/tm5/observationoperator.py

@@ -323,7 +323,7 @@ class TM5ObservationOperator(ObservationOperator):
             raise IOError,msg
 
 
-        for n in range(int(self.DaCycle['forecast.nmembers'])):
+        for n in range(int(self.DaCycle['da.optimizer.nmembers'])):
             paramfile = 'parameters.%03d.nc'%n
             if paramfile not in datafiles:
                 msg = "The specified parameter input file for the TM5 model to read from does not exist (%s), exiting..."%paramfile ; logging.error(msg)
@@ -459,7 +459,7 @@ class TM5ObservationOperator(ObservationOperator):
         if os.path.exists(okfile): 
             dummy               = os.remove(okfile)
 
-        nprocesses              = self.DaCycle['forecast.nmembers']
+        nprocesses              = self.DaCycle['da.optimizer.nmembers']
         jobparams               = {'jobname':'tm5',
                                    'jobnodes':'ncomp %d'%int(nprocesses),
                                    'jobtime':'00:30:00',
@@ -515,7 +515,7 @@ class TM5ObservationOperator(ObservationOperator):
         jobfile                 = os.path.join(targetdir,'jb.%s.jb'%jobid)
         logfile                 = jobfile.replace('.jb','.log')
 
-        nprocesses              = int(self.DaCycle['forecast.nmembers'])/5  # Note that we assign 5 tracers to each processor, this seems good for TM5
+        nprocesses              = int(self.DaCycle['da.optimizer.nmembers'])/5  # Note that we assign 5 tracers to each processor, this seems good for TM5
         jobparams               = {'jobname':'tm5',
                                    'jobnodes':'ncomp %d'%int(nprocesses),
                                    'jobtime':'00:30:00',

da/tools/initexit.py

@@ -19,24 +19,23 @@ like this:::
 
     ! Info on the data assimilation cycle
 
-    time.restart        : False
-    time.start          : 2000-01-01 00:00:00
-    time.finish         : 2000-01-08 00:00:00
-    time.cycle          : 7
-    time.nlag           : 2
-    dir.da_run          : ${HOME}/tmp/test_da
+    time.restart        : False                     ! Restart from an existing run T/F
+    time.start          : 2000-01-01 00:00:00       ! Start time of first cycle
+    time.finish         : 2000-01-08 00:00:00       ! End time of last cycle
+    time.cycle          : 7                         ! length of each cycle, 7 means one week
+    time.nlag           : 5                         ! number of cycles in one smoother window
+    dir.da_run          : ${HOME}/tmp/test_da       ! the run directory for you project
 
     ! Info on the DA system used
 
-    da.system           : CarbonTracker
-    da.system.rc        : carbontracker.rc
-    da.platform         : maunaloa
+    da.system           : CarbonTracker             ! an identifier for your inversion system
+    da.system.rc        : da/rc/carbontracker.rc    ! the settings needed in your inversion system
 
     ! Info on the forward model to be used
 
-    forecast.model      : TM5
-    forecast.model.rc   : ${HOME}/Modeling/TM5/ct_new.rc
-    forecast.nmembers   : 2
+    da.obsoperator         : TM5                                ! an identifier for your observation operator
+    da.obsoperator.rc      : ${HOME}/Modeling/TM5/tm5-ctdas.rc  ! the rc-file needed to run youobservation operator
+    da.optimizer.nmembers  : 30                                 ! the number of ensemble members desired in the optimization
 
 The most important method of the CycleControl object are listed below:
 
@@ -62,7 +61,11 @@ needed_da_items=[
     'time.nlag',
     'time.cycle',
     'dir.da_run',
-    'forecast.nmembers']
+    'da.system',
+    'da.system.rc',
+    'da.obsoperator',
+    'da.obsoperator.rc',
+    'da.optimizer.nmembers']
 
 # only needed in an earlier implemented where each substep was a separate job
 # validprocesses = ['start','done','samplestate','advance','invert']
@@ -115,7 +118,7 @@ class CycleControl(dict):
         msg = "DA Cycle rc-file is %s" % self.RcFileName                                ; print msg
         msg = "DA Cycle run directory is %s" % self['dir.da_run']           ; print msg
         msg = "DA Cycle inverse system is %s" % self['da.system']           ; print msg
-        msg = "DA Cycle forecast model is %s" % self['forecast.model']      ; print msg
+        msg = "DA Cycle obs operator is %s" % self['da.obsoperator']        ; print msg
         msg = "==============================================================="    ; print msg
 
         return ""
@@ -161,9 +164,13 @@ class CycleControl(dict):
         for key in needed_da_items:
 
             if not self.has_key(key):
-                status,msg = ( False,'Missing a required value in rc-file : %s' % key)
-                logging.error(msg)
-                raise IOError,msg
+                status,msge = ( False,'Missing a required value in rc-file : %s' % key)  
+                logging.error(msge)
+                msg = '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ' ; logging.error(msg)
+                msg = 'Please note the update on Dec 02 2011 where rc-file names for DaSystem and ' ; logging.error(msg)
+                msg = 'are from now on specified in the main rc-file (see da/rc/da.rc for example)' ; logging.error(msg)
+                msg = '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ' ; logging.error(msg)
+                raise IOError,msge
 
         status,msg = ( True,'DA Cycle settings have been validated succesfully' )  ; logging.debug(msg)
 

da/tools/pipeline.py

@@ -258,7 +258,7 @@ def Invert(DaCycle, StateVector, Optimizer ):
     """ Perform the inverse calculation """
 
     dims      = ( int(DaCycle['time.nlag']),
-                  int(DaCycle['forecast.nmembers']),
+                  int(DaCycle['da.optimizer.nmembers']),
                   int(DaCycle.DaSystem['nparameters']),
                   StateVector.nobs,  )
 

das.py

@@ -46,8 +46,8 @@ from da.tm5.observationoperator import TM5ObservationOperator
 from da.ct.optimizer import CtOptimizer
 
 PlatForm    = MaunaloaPlatForm()
-DaSystem    = CtDaSystem('da/rc/carbontracker.rc')
-ObsOperator = TM5ObservationOperator('/Users/peters/Modeling/TM5/tm5-ctdas-inv-ei-6x4.rc')
+DaSystem    = CtDaSystem(DaCycle['da.system.rc'])
+ObsOperator = TM5ObservationOperator(DaCycle['da.obsoperator.rc'])
 Samples     = CtObservations()
 StateVector = CtStateVector()
 Optimizer   = CtOptimizer()

dasjet.py

@@ -46,8 +46,8 @@ from da.tm5.observationoperator import TM5ObservationOperator
 from da.ct.optimizer import CtOptimizer
 
 PlatForm    = JetPlatForm()
-DaSystem    = CtDaSystem('da/rc/carbontrackerjet.rc')
-ObsOperator = TM5ObservationOperator('/home/peters/TM/TM5_new/tm5-ctdas-inv-tmpp.rc')
+DaSystem    = CtDaSystem(DaCycle['da.system.rc'])
+ObsOperator = TM5ObservationOperator(DaCycle['da.obsoperator.rc'])
 Samples     = CtObservations()
 StateVector = CtStateVector()
 Optimizer   = CtOptimizer()