diff --git a/da/baseclasses/dasystem.py b/da/baseclasses/dasystem.py
index 6dfa85ba406762a1e70138b797488b9671494580..f57bac465bc81cdf327e8e192b77eef9740af04a 100755
--- a/da/baseclasses/dasystem.py
+++ b/da/baseclasses/dasystem.py
@@ -10,22 +10,22 @@ File created on 26 Aug 2010.
The DaSystem class is found in the module :mod:`dasystem`, or in a specific implementation under the da/ source tree. It is derived from the standard python :class:`dictionary` object.
-It describes the details of the data assimilation system used (i.e., CarbonTracker, or CT Methane, or ....).
-
-An example rc-file for CarbonTracker is:::
-
- !!! Info for the CarbonTracker data assimilation system
-
- datadir : /data/CO2/carbontracker/ct08/
- obs.input.dir : ${datadir}/obsnc/with_fillvalue/
- obs.input.fname : obs_final.nc
- ocn.covariance : ${datadir}/oif_p3_era40.dpco2.2000.01.hdf
- bio.covariance : ${datadir}/covariance_bio_olson19.nc
- deltaco2.prefix : oif_p3_era40.dpco2
- regtype : olson19_oif30
- nparameters : 240
- random.seed : 4385
- regionsfile : transcom_olson19_oif30.hdf
+It describes the details of the data assimilation system used (i.e., CarbonTracker, or CT Methane, or ....) ::
+
+ datadir : /Volumes/Storage/CO2/carbontracker/input/ct08/ ! The directory where input data is found
+ obs.input.dir : ${datadir}/obsnc/with_fillvalue ! the observation input dir
+ obs.input.fname : obs_forecast.nc ! the observation input file
+ ocn.covariance : ${datadir}/oif_p3_era40.dpco2.2000.01.hdf ! the ocean flux covariance file
+ bio.covariance : ${datadir}/covariance_bio_olson19.nc ! the biosphere flux covariance file
+ deltaco2.prefix : oif_p3_era40.dpco2 ! the type of ocean product used
+ regtype : olson19_oif30 ! the ecoregion definitions
+ nparameters : 240 ! the number of parameters to solve for
+ random.seed : 4385 ! the random seed for the first cycle
+ regionsfile : transcom_olson19_oif30.hdf ! the ecoregion defintion mask file
+
+ ! Info on the sites file used
+
+ obs.sites.rc : ${datadir}/sites_and_weights_co2.ct10.rc ! the weights in the covariance matric of each obs
The full baseclass description:
diff --git a/da/baseclasses/optimizer.py b/da/baseclasses/optimizer.py
index cc23245b9cfd560777aad2a9bdf661d2fbf247bf..4601447aaa75b3a71c1cce434f817ac5ada8b41f 100755
--- a/da/baseclasses/optimizer.py
+++ b/da/baseclasses/optimizer.py
@@ -458,7 +458,7 @@ if __name__ == "__main__":
nobs = len(samples.Data)
dims = ( int(DaCycle['time.nlag']),
- int(DaCycle['forecast.nmembers']),
+ int(DaCycle['da.optimizer.nmembers']),
int(DaCycle.DaSystem['nparameters']),
nobs, )
diff --git a/da/baseclasses/statevector.py b/da/baseclasses/statevector.py
index 4eeb82735f3dbb468a1e5105d42f621bb9c8e938..6f930736979ebd717f803e2b856cb5d31d52f319 100755
--- a/da/baseclasses/statevector.py
+++ b/da/baseclasses/statevector.py
@@ -190,7 +190,7 @@ class StateVector(object):
"""
dims = ( int(self.DaCycle['time.nlag']),
- int(self.DaCycle['forecast.nmembers']),
+ int(self.DaCycle['da.optimizer.nmembers']),
int(self.DaCycle.DaSystem['nparameters']),
)
@@ -423,7 +423,7 @@ if __name__ == "__main__":
print DaCycle
dims = ( int(DaCycle['time.nlag']),
- int(DaCycle['forecast.nmembers']),
+ int(DaCycle['da.optimizer.nmembers']),
int(DaCycle.DaSystem['nparameters']),
)
diff --git a/da/ct/optimizer.py b/da/ct/optimizer.py
index b1746e5d63bafb52291371d5a5640f0ede518fcb..cd4bb68220b51f9fc33516bb9609482c428407a1 100755
--- a/da/ct/optimizer.py
+++ b/da/ct/optimizer.py
@@ -91,7 +91,7 @@ if __name__ == "__main__":
nobs = 100
dims = ( int(DaCycle['time.nlag']),
- int(DaCycle['forecast.nmembers']),
+ int(DaCycle['da.optimizer.nmembers']),
int(DaCycle.DaSystem['nparameters']),
nobs, )
diff --git a/da/doc/source/tut_chapter1.rst b/da/doc/source/tut_chapter1.rst
index 47000065efacec9725b0f31cad4b2073f0cda386..99d8de1f88ec74a99db6b2c769d5a885a4f64322 100644
--- a/da/doc/source/tut_chapter1.rst
+++ b/da/doc/source/tut_chapter1.rst
@@ -5,11 +5,11 @@ Chapter 1: Configuring CTDAS to run an experiment
**In which you will learn to modify the three primary files that together control a CTDAS experiment**
-Modify the two primary rc-files
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Step 1: Modify the two primary rc-files
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
When you checkout CTDAS from the subversion server, there are two example rc-files that you need to modify.
- 1. The ``das.rc`` file, which describes your CTDAS configuration with respect to experiment name, time period, and lag
+ 1. The ``da.rc`` file, which describes your CTDAS configuration with respect to experiment name, time period, and lag
2. The ``carbontracker.rc`` file, which describes your CTDAS configuration with respect to observations and statevector
.. note:: The example files are found in the da/rc/ directory of your CTDAS tree. You are encouraged to create copies of these
@@ -28,14 +28,14 @@ can be replaced by: ::
dir.da_run : /scratch/${USER}/my_first_ctdas_run
Which, as you have likely guessed, will change the location where CTDAS creates a directory structure and places input/output files
-for your simulation.
+for your simulation. See :mod:`initexit` for more information on these settings. It is especially important to set the keys ::
-Where the ``das.rc`` file is rather self-explanatory, the ``carbontracker.rc`` file has keys that refer to the inner workings of CTDAS,
-such as: ::
+ da.system.rc : da/rc/carbontracker.rc ! the settings needed in your inversion system
+ da.obsoperator.rc : ${HOME}/Modeling/TM5/tm5-ctdas.rc ! the rc-file needed to run youobservation operator
- ocn.covariance : ${datadir}/oif_p3_era40.dpco2.2000.01.hdf
- regionsfile : transcom_olson19_oif30.hdf
- obs.sites.rc : ${datadir}/sites_and_weights_co2.ct10.rc
+correctly. The first one refers to the rc-file (2) described above, while the second one refers to the rc-file you used to compile the TM5 model in :ref:`Chapter 0 <tut_chapter0>`.
+
+Where the ``da.rc`` file is rather self-explanatory, the ``carbontracker.rc`` file has keys that refer to the inner workings of CTDAS as described in :mod:`dasystem`
These specify files, or paths to files, that are needed to construct the :ref:`statevector <Statevector>` object and the
:ref:`observations <Observation>` object. For instance, they allow CTDAS to map parameters to the global grid, or to
@@ -45,15 +45,15 @@ construct model-data-mismatches for the sites that will be used in the assimilat
fact that they are missing.
-Modify the primary run script
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Step 2: Modify the python run script
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-The primary run script for CTDAS is simply called ``das.py`` and can also be found in the ``da/examples/`` subdirectory. As before,
-you should copy this file before modyfing it.
+You are required to create your own python run script for CTDAS. In this tutorial, we assume it is simply called ``das.py`` and an example
+is checked out in the ``da/tools/`` trunk directory. As before, you should copy this file before modyfing it.
.. note:: The primary run script needs to be located in the main directory of the CTDAS tree, i.e., ``${yourdir}/da/ct/trunk/``
-The header of this script can contain settings that are needed to run it on your platform (e.g., queue flags), and is then followed by
+The script first initializes some python objects needed to log activity and to parse command line arguments, and is then followed by
a block where all the modules that are needed in your experiment are imported. The example below shows the import of several classes that
are needed to run the CarbonTracker CO2 system on a computer referred to as MaunaLoa, using TM5 as
an :ref:`observationoperator <Observation Operator>` ::
@@ -69,28 +69,30 @@ an :ref:`observationoperator <Observation Operator>` ::
from da.tm5.observationoperator import TM5ObservationOperator
from da.ct.optimizer import CtOptimizer
-Once the classes are loaded successfully, the objects can be created. Note how the 2nd and 3rd object make use of rc-files! ::
+Once the classes are loaded successfully, the objects are created. ::
PlatForm = MaunaloaPlatForm()
- DaSystem = CtDaSystem('carbontracker.rc')
- ObsOperator = TM5ObservationOperator('/Users/peters/Modeling/TM5/tm5-ctdas.rc')
+ DaSystem = CtDaSystem(DaCycle['da.system.rc'])
+ ObsOperator = TM5ObservationOperator(DaCycle['da.obsoperator.rc'])
Samples = CtObservations()
StateVector = CtStateVector()
Optimizer = CtOptimizer()
+.. note:: See how the initilization of the DaSystem and ObservationOperator object make use of the keys specified in your primary rc-file !
+
Modification of these objects might be desirable for more advanced users, and in case of the :ref:`platform <Platform>` object, even
necessary (see next section). Once the objects are created, they are simply passed to a pipeline for the CTDAS. In the first
chapter of the tutorial, we will assume this pipeline is immutable.
-Thus, once you have modified the headers of the ``das.py`` script, and changed the references to the rc-files, you can focus on the last
-configuration step:
+In order to complete the preparations for your first run, you might have to also go through Step 3:
-Creating a PlatForm object for your system
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Step 3: Creating a PlatForm object for your system
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
From the description of the :ref:`platform <Platform>` object, you will understand that this object is (partly) unique for each user,
or at least for the computing environment of each user. Information on the computing system is therefore coded into a specific python object.
-**This object will need to be created for your system by you.**
+
+.. warning:: **This object will need to be created for your system by you.**
Luckily, part of the work is already done. In the ``da/baseclasses`` subdirectory you will find a baseclass :ref:`platform <Platform>`
which serves as a blueprint for your own ``Platform`` object. This is done through class inheritance. As an example, you can open one
@@ -128,13 +130,14 @@ While the baseclass did not have any functionality, a call to the ``GetJobTempla
a job script on NOAA's "jet" supercomputer, so that we can submit jobs to its queue. **By modifying each of the methods in your own
derived PlatForm class in the same way, you can make each method work on your system**.
+.. note:: As default, you could first try if the simple baseclass PlatForm works for you
+
Once you have created your own PlatForm object, and you have successfully imported and instantiated it in your primary python run script,
you are ready to give your first experiment a try.
+
.. note:: Sometimes it is faster and easier to test your newly created class 'offline'. At the end of your module, following the __main__
section you can add lines to test your PlatForm object before plugging it into the CTDAS.
-.. note:: I am hoping to one day implement simple Test() functions for all methods so that code can be formally tested before
- using it in a real world application.
diff --git a/da/doc/source/tut_chapter2.rst b/da/doc/source/tut_chapter2.rst
index b7480d832c481537fa1bf25354762eb90e7f1048..a7c243774e042b05e7204ca39b653eb209d13678 100644
--- a/da/doc/source/tut_chapter2.rst
+++ b/da/doc/source/tut_chapter2.rst
@@ -1,4 +1,4 @@
-.. _tut_chapter1:
+.. _tut_chapter2:
Chapter 2: Running your first experiment
----------------------------------------------------
diff --git a/da/tm5/observationoperator.py b/da/tm5/observationoperator.py
index c44d958b9022fd427b3a242d4e315f62100a44d8..5784d1964d9e9fca6dc175c6f4924bbabff62869 100755
--- a/da/tm5/observationoperator.py
+++ b/da/tm5/observationoperator.py
@@ -323,7 +323,7 @@ class TM5ObservationOperator(ObservationOperator):
raise IOError,msg
- for n in range(int(self.DaCycle['forecast.nmembers'])):
+ for n in range(int(self.DaCycle['da.optimizer.nmembers'])):
paramfile = 'parameters.%03d.nc'%n
if paramfile not in datafiles:
msg = "The specified parameter input file for the TM5 model to read from does not exist (%s), exiting..."%paramfile ; logging.error(msg)
@@ -459,7 +459,7 @@ class TM5ObservationOperator(ObservationOperator):
if os.path.exists(okfile):
dummy = os.remove(okfile)
- nprocesses = self.DaCycle['forecast.nmembers']
+ nprocesses = self.DaCycle['da.optimizer.nmembers']
jobparams = {'jobname':'tm5',
'jobnodes':'ncomp %d'%int(nprocesses),
'jobtime':'00:30:00',
@@ -515,7 +515,7 @@ class TM5ObservationOperator(ObservationOperator):
jobfile = os.path.join(targetdir,'jb.%s.jb'%jobid)
logfile = jobfile.replace('.jb','.log')
- nprocesses = int(self.DaCycle['forecast.nmembers'])/5 # Note that we assign 5 tracers to each processor, this seems good for TM5
+ nprocesses = int(self.DaCycle['da.optimizer.nmembers'])/5 # Note that we assign 5 tracers to each processor, this seems good for TM5
jobparams = {'jobname':'tm5',
'jobnodes':'ncomp %d'%int(nprocesses),
'jobtime':'00:30:00',
diff --git a/da/tools/initexit.py b/da/tools/initexit.py
index 1ad4a6197585f3ec7cc7e1bb4158819600d04121..9d40d4aa62b273460fccba345a99b37f16507862 100755
--- a/da/tools/initexit.py
+++ b/da/tools/initexit.py
@@ -19,24 +19,23 @@ like this:::
! Info on the data assimilation cycle
- time.restart : False
- time.start : 2000-01-01 00:00:00
- time.finish : 2000-01-08 00:00:00
- time.cycle : 7
- time.nlag : 2
- dir.da_run : ${HOME}/tmp/test_da
+ time.restart : False ! Restart from an existing run T/F
+ time.start : 2000-01-01 00:00:00 ! Start time of first cycle
+ time.finish : 2000-01-08 00:00:00 ! End time of last cycle
+ time.cycle : 7 ! length of each cycle, 7 means one week
+ time.nlag : 5 ! number of cycles in one smoother window
+ dir.da_run : ${HOME}/tmp/test_da ! the run directory for you project
! Info on the DA system used
- da.system : CarbonTracker
- da.system.rc : carbontracker.rc
- da.platform : maunaloa
+ da.system : CarbonTracker ! an identifier for your inversion system
+ da.system.rc : da/rc/carbontracker.rc ! the settings needed in your inversion system
! Info on the forward model to be used
- forecast.model : TM5
- forecast.model.rc : ${HOME}/Modeling/TM5/ct_new.rc
- forecast.nmembers : 2
+ da.obsoperator : TM5 ! an identifier for your observation operator
+ da.obsoperator.rc : ${HOME}/Modeling/TM5/tm5-ctdas.rc ! the rc-file needed to run youobservation operator
+ da.optimizer.nmembers : 30 ! the number of ensemble members desired in the optimization
The most important method of the CycleControl object are listed below:
@@ -62,7 +61,11 @@ needed_da_items=[
'time.nlag',
'time.cycle',
'dir.da_run',
- 'forecast.nmembers']
+ 'da.system',
+ 'da.system.rc',
+ 'da.obsoperator',
+ 'da.obsoperator.rc',
+ 'da.optimizer.nmembers']
# only needed in an earlier implemented where each substep was a separate job
# validprocesses = ['start','done','samplestate','advance','invert']
@@ -115,7 +118,7 @@ class CycleControl(dict):
msg = "DA Cycle rc-file is %s" % self.RcFileName ; print msg
msg = "DA Cycle run directory is %s" % self['dir.da_run'] ; print msg
msg = "DA Cycle inverse system is %s" % self['da.system'] ; print msg
- msg = "DA Cycle forecast model is %s" % self['forecast.model'] ; print msg
+ msg = "DA Cycle obs operator is %s" % self['da.obsoperator'] ; print msg
msg = "===============================================================" ; print msg
return ""
@@ -161,9 +164,13 @@ class CycleControl(dict):
for key in needed_da_items:
if not self.has_key(key):
- status,msg = ( False,'Missing a required value in rc-file : %s' % key)
- logging.error(msg)
- raise IOError,msg
+ status,msge = ( False,'Missing a required value in rc-file : %s' % key)
+ logging.error(msge)
+ msg = '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ' ; logging.error(msg)
+ msg = 'Please note the update on Dec 02 2011 where rc-file names for DaSystem and ' ; logging.error(msg)
+ msg = 'are from now on specified in the main rc-file (see da/rc/da.rc for example)' ; logging.error(msg)
+ msg = '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ' ; logging.error(msg)
+ raise IOError,msge
status,msg = ( True,'DA Cycle settings have been validated succesfully' ) ; logging.debug(msg)
diff --git a/da/tools/pipeline.py b/da/tools/pipeline.py
index 80b918399b83dd8a3761cd19575184427eb4f5da..aadfd3286ec537e58ed04a17e174c9fc3757ddf5 100755
--- a/da/tools/pipeline.py
+++ b/da/tools/pipeline.py
@@ -258,7 +258,7 @@ def Invert(DaCycle, StateVector, Optimizer ):
""" Perform the inverse calculation """
dims = ( int(DaCycle['time.nlag']),
- int(DaCycle['forecast.nmembers']),
+ int(DaCycle['da.optimizer.nmembers']),
int(DaCycle.DaSystem['nparameters']),
StateVector.nobs, )
diff --git a/das.py b/das.py
index b0bca4b32bbfadd2a4f6123bef95520323280f0c..0490285dae8ecc4219932d821e51dd3349c9bf81 100755
--- a/das.py
+++ b/das.py
@@ -46,8 +46,8 @@ from da.tm5.observationoperator import TM5ObservationOperator
from da.ct.optimizer import CtOptimizer
PlatForm = MaunaloaPlatForm()
-DaSystem = CtDaSystem('da/rc/carbontracker.rc')
-ObsOperator = TM5ObservationOperator('/Users/peters/Modeling/TM5/tm5-ctdas-inv-ei-6x4.rc')
+DaSystem = CtDaSystem(DaCycle['da.system.rc'])
+ObsOperator = TM5ObservationOperator(DaCycle['da.obsoperator.rc'])
Samples = CtObservations()
StateVector = CtStateVector()
Optimizer = CtOptimizer()
diff --git a/dasjet.py b/dasjet.py
index 0c028eb8da631df991bdb15a4c7f6d9e2d9e4003..e991aa6fd073c9ffe8c227c330d6a2e0f42ac580 100755
--- a/dasjet.py
+++ b/dasjet.py
@@ -46,8 +46,8 @@ from da.tm5.observationoperator import TM5ObservationOperator
from da.ct.optimizer import CtOptimizer
PlatForm = JetPlatForm()
-DaSystem = CtDaSystem('da/rc/carbontrackerjet.rc')
-ObsOperator = TM5ObservationOperator('/home/peters/TM/TM5_new/tm5-ctdas-inv-tmpp.rc')
+DaSystem = CtDaSystem(DaCycle['da.system.rc'])
+ObsOperator = TM5ObservationOperator(DaCycle['da.obsoperator.rc'])
Samples = CtObservations()
StateVector = CtStateVector()
Optimizer = CtOptimizer()