next effort on writing documentation started

da/baseclasses/dasystem.py

@@ -7,6 +7,30 @@ Author : peters
 Revision History:
 File created on 26 Aug 2010.
 
+The DaSystem class is found in the module :mod:`da.baseclasses.dasystem`, or in a specific implementation under the da/ source tree. It is derived from the standard python :class:`dictionary` object. 
+
+It describes the details of the data assimilation system used (i.e., CarbonTracker, or CT Methane, or ....).
+
+An example rc-file for CarbonTracker is:::
+
+    !!! Info for the CarbonTracker data assimilation system
+
+    datadir         : /data/CO2/carbontracker/ct08/
+    obs.input.dir   : ${datadir}/obsnc/with_fillvalue/
+    obs.input.fname : obs_final.nc
+    ocn.covariance  : ${datadir}/oif_p3_era40.dpco2.2000.01.hdf 
+    bio.covariance  : ${datadir}/covariance_bio_olson19.nc
+    deltaco2.prefix : oif_p3_era40.dpco2
+    regtype         : olson19_oif30
+    nparameters     : 240
+    random.seed     : 4385
+    regionsfile     : transcom_olson19_oif30.hdf
+
+The full baseclass description:
+
+.. autoclass:: da.baseclasses.dasystem.DaSystem
+   :members:
+
 """
 
 import os

da/baseclasses/obs.py

@@ -7,11 +7,18 @@ Author : peters
 Revision History:
 File created on 28 Jul 2010.
 
+.. autoclass:: da.baseclasses.obs.Observation 
+   :members: Initialize, Validate, AddObs, AddSimulations, AddModelDataMismatch, WriteSampleInfo  
+
+.. autoclass:: da.baseclasses.obs.ObservationList 
+   :members: __init__
+
 """
 import os
 import sys
 import logging
 import datetime
+from numpy import array, ndarray
 
 identifier = 'Observation baseclass'
 version    = '0.0'
@@ -19,9 +26,30 @@ version    = '0.0'
 ################### Begin Class Observation ###################
 
 class Observation(object):
-    """ an object that holds data + methods and attributes needed to manipulate observations for a DA cycle """
+    """ 
+    The baseclass Observation is a generic object that provides a number of methods required for any type of observations used in 
+    a data assimilation system. These methods are called from the CarbonTracker pipeline. 
+
+    .. note:: Most of the actual functionality will need to be provided through a derived Observations class with the methods 
+              below overwritten. Writing your own derived class for Observations is one of the first tasks you'll likely 
+              perform when extending or modifying the CarbonTracker Data Assimilation Shell.
+
+    Upon initialization of the class, an object is created that holds no actual data, but has a placeholder attribute `self.Data` 
+    which is an empty list of type :class:`~da.baseclasses.obs.ObservationList`. An ObservationList object is created when the 
+    method :meth:`~da.baseclasses.obs.Observation.AddObs` is invoked in the pipeline. 
+
+    From the list of observations, a file is written  by method 
+    :meth:`~da.baseclasses.obs.Observation.WriteSampleInfo`
+    with the sample info needed by the 
+    :class:`~da.baseclasses.observationoperator.ObservationOperator` object. The values returned after sampling 
+    are finally added by :meth:`~da.baseclasses.obs.Observation.AddSimulations`
+
+    """ 
 
     def __init__(self):
+        """
+        create an object with an identifier, version, and an empty ObservationList
+        """
         self.Identifier = self.getid()
         self.Version    = self.getversion()
         self.Data       = ObservationList([])  # Initialize with an empty list of obs
@@ -29,9 +57,15 @@ class Observation(object):
         msg                 = 'Observation object initialized: %s'%self.Identifier ; logging.info(msg)
 
     def getid(self):
+        """
+        Return the identifier of the Observation Object, used for logging purposed
+        """
         return identifier
 
     def getversion(self):
+        """
+        Return the version of the Observation Object, used for logging purposed
+        """
         return identifier
 
     def __str__(self):
@@ -49,8 +83,12 @@ class Observation(object):
         """
 
     def AddObs(self):
-        """ Add the observation data to the Observation object. This is in a form of a list that goes under the name Data. The
-            list has as only requirement that it can return the observed+simulated values through a method "getvalues"
+        """ 
+        Add actual observation data to the Observation object. This is in a form of an 
+        :class:`~da.baseclasses.obs.ObservationList` that is contained in self.Data. The 
+        list has as only requirement that it can return the observed+simulated values 
+        through the method :meth:`~da.baseclasses.obs.ObservationList.getvalues`
+
         """
 
     def AddSimulations(self):
@@ -76,15 +114,14 @@ class Observation(object):
 
 class ObservationList(list):
     """ This is a special type of list that holds observed sample objects. It has methods to extract data from such a list easily """
-    from numpy import array, ndarray
 
     def getvalues(self,name,constructor=array):
-            from numpy import ndarray
-            result = constructor([getattr(o,name) for o in self])
-            if isinstance(result,ndarray): 
-                return result.squeeze()
-            else:
-                return result
-
-################### End Class MixingRatioList ###################
+
+        result = constructor([getattr(o,name) for o in self])
+        if isinstance(result,ndarray): 
+            return result.squeeze()
+        else:
+            return result
+
+################### End Class ObservationList ###################
 

da/baseclasses/platform.py

@@ -7,6 +7,17 @@ Author : peters
 Revision History:
 File created on 06 Sep 2010.
 
+The PlatForm class is found in the module :mod:`da.baseclasses.platform`, or in a specific implementation under the da/source tree. 
+
+The platform object holds attributes and methods that allow job control on each specific platform. This includes methods to create and submit jobs, but also to obtain process and/or job ID's. These are needed to control the flow of 
+the system on each platform.
+
+Typically, every platform needs specific implementations of this object (through inheritance), and you should refer to your specific PlatForm object documentation for details (see *da/platform/*).
+
+.. autoclass:: da.baseclasses.platform.PlatForm
+   :members:
+   :inherited-members:
+
 """
 
 import sys

da/doc/Makefile

@@ -3,7 +3,7 @@
 
 # You can set these variables from the command line.
 SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
+SPHINXBUILD   = sphinx-build-2.6
 PAPER         =
 BUILDDIR      = build
 

da/doc/source/cyclecontrol.rst

@@ -2,42 +2,4 @@
 
 Cycle Control
 ======================
-The CycleControl class is found in the module :mod:`da.tools.initexit`. It is derived from the standard python :class:`dictionary` object. It is the only core object of CTDAS that is automatically created in the pipeline, the user (normally) does not need to modify or extend it. The class is created based on options and arguments passes on the command line when submitting your main CTDAS job. 
-
-Valid options are defined in 
-
-.. autofunction:: da.tools.initexit.ParseOptions
-
-With the name of a valid ``rc-file``, the CycleControl object is instantiated and validated. An example rc-file looks
-like this:::
-
-    ! Info on the data assimilation cycle
-
-    time.restart        : False
-    time.start          : 2000-01-01 00:00:00
-    time.finish         : 2000-01-08 00:00:00
-    time.cycle          : 7
-    time.nlag           : 2
-    dir.da_run          : ${HOME}/tmp/test_da
-
-    ! Info on the DA system used
-
-    da.system           : CarbonTracker
-    da.system.rc        : carbontracker.rc
-    da.platform         : maunaloa
-
-    ! Info on the forward model to be used
-
-    forecast.model      : TM5
-    forecast.model.rc   : ${HOME}/Modeling/TM5/ct_new.rc
-    forecast.nmembers   : 2
-
-The most important method of the CycleControl object are listed below:
-
-.. autoclass:: da.tools.initexit.CycleControl 
-   :members: Initialize, Finalize, SubmitNextCycle, CollectSaveData, MoveSaveData, SetupFileStructure, RecoverRun, RandomSeed
-
-Two important attributes of the CycleControl object are:
-    (1) DaSystem, an instance of a :ref:`dasystem`
-    (2) DaPlatForm, an instance of a :ref:`platform`
-
+.. automodule:: da.tools.initexit

da/doc/source/dasystem.rst

@@ -2,27 +2,4 @@
 
 DaSystem
 ======================
-The DaSystem class is found in the module :mod:`da.baseclasses.dasystem`, or in a specific implementation under the da/ source tree. It is derived from the standard python :class:`dictionary` object. 
-
-It describes the details of the data assimilation system used (i.e., CarbonTracker, or CT Methane, or ....).
-
-An example rc-file for CarbonTracker is:::
-
-    !!! Info for the CarbonTracker data assimilation system
-
-    datadir         : /data/CO2/carbontracker/ct08/
-    obs.input.dir   : ${datadir}/obsnc/with_fillvalue/
-    obs.input.fname : obs_final.nc
-    ocn.covariance  : ${datadir}/oif_p3_era40.dpco2.2000.01.hdf 
-    bio.covariance  : ${datadir}/covariance_bio_olson19.nc
-    deltaco2.prefix : oif_p3_era40.dpco2
-    regtype         : olson19_oif30
-    nparameters     : 240
-    random.seed     : 4385
-    regionsfile     : transcom_olson19_oif30.hdf
-
-
-The full baseclass description:
-
-.. autoclass:: da.baseclasses.dasystem.DaSystem
-   :members: __init__
+.. automodule:: da.baseclasses.dasystem

da/doc/source/documentation.rst

@@ -10,4 +10,8 @@ Documentation
    The CycleControl Object <cyclecontrol>
    The DASystem Object <dasystem>
    The PlatForm Object <platform>
-
+   The Observations Object <observations>
+       The CarbonTracker Observations Object <observations>
+   The ObservationOperator Object <observationoperator>
+   The StateVector Object <statevector>
+   The Optimizer Object <optimizer>

da/doc/source/platform.rst

@@ -2,15 +2,5 @@
 
 PlatForm
 ======================
-The PlatForm class is found in the module :mod:`da.baseclasses.platform`, or in a specific implementation under the da/source tree. 
 
-The platform object holds attributes and methods that allow job control on each specific platform. This includes methods to create and submit jobs, but also to obtain process and/or job ID's. These are needed to control the flow of 
-the system on each platform.
-
-Typically, every platform needs specific implementations of this object (through inheritance), and you should refer to your specific PlatForm object documentation for details (see *da/platform/*).
-
-We give here the class description of the baseclass platform:
-
-.. autoclass:: da.baseclasses.platform.PlatForm
-   :members:
-   :inherited-members:
+.. automodule:: da.baseclasses.platform

da/tools/initexit.py

@@ -7,6 +7,51 @@ Author : peters
 Revision History:
 File created on 13 May 2009.
 
+The CycleControl class is found in the module :mod:`da.tools.initexit`. It is derived from the standard python :class:`dictionary` object. It is the only core object of CTDAS that is automatically created in the pipeline, the user (normally) does not need to modify or extend it. The class is created based on options and arguments passes on the command line when submitting your main CTDAS job. 
+
+Valid options are defined in 
+
+.. autofunction:: da.tools.initexit.ParseOptions
+
+With the name of a valid ``rc-file``, the CycleControl object is instantiated and validated. An example rc-file looks
+like this:::
+
+    ! Info on the data assimilation cycle
+
+    time.restart        : False
+    time.start          : 2000-01-01 00:00:00
+    time.finish         : 2000-01-08 00:00:00
+    time.cycle          : 7
+    time.nlag           : 2
+    dir.da_run          : ${HOME}/tmp/test_da
+
+    ! Info on the DA system used
+
+    da.system           : CarbonTracker
+    da.system.rc        : carbontracker.rc
+    da.platform         : maunaloa
+
+    ! Info on the forward model to be used
+
+    forecast.model      : TM5
+    forecast.model.rc   : ${HOME}/Modeling/TM5/ct_new.rc
+    forecast.nmembers   : 2
+
+The most important method of the CycleControl object are listed below:
+
+.. autoclass:: da.tools.initexit.CycleControl 
+   :members: Initialize, Finalize, SubmitNextCycle, CleanUpCycle, SetupFileStructure, RecoverRun, RandomSeed
+
+Two important attributes of the CycleControl object are:
+    (1) DaSystem, an instance of a :ref:`dasystem`
+    (2) DaPlatForm, an instance of a :ref:`platform`
+
+Other functions in the module initexit that are related to the control of a DA cycle are:
+
+.. autofunction:: da.tools.initexit.StartLogger 
+.. autofunction:: da.tools.initexit.ValidateOptsArgs 
+
+
 """
 
 needed_da_items=[
@@ -543,6 +588,23 @@ class CycleControl(dict):
 
         return None
 
+    def CleanUpCycle(self):
+        """
+        Move the log file, clean up rundir after a cycle has finished
+        """
+
+        # move log file to rundir/jobs
+        jobdir      = os.path.join(self['dir.da_run'],"jobs")
+        logfile     = 'das.o%s'%self.DaPlatForm.GetMyID()
+
+        if os.path.exists(logfile):
+            joblogfile  = os.path.join(jobdir,'cycle.%s.log'%self['time.sample.start'].strftime('%Y%m%d')) 
+            dummy       = shutil.move(logfile,joblogfile)
+            msg         = "....Moved %s to %s"%(logfile,joblogfile)   ; logging.debug(msg)
+
+            msg = "The complete log file is now at: %s"%(joblogfile)   ; logging.info(msg)
+
+
 
 def StartLogger():
     """ start the logging of messages to screen"""
@@ -636,22 +698,6 @@ def ValidateOptsArgs(opts,args):
 
     return opts,args
 
-def CleanUpCycle(DaCycle):
-    """
-    Move files, clean up rundir after a cycle has finished
-    """
-
-    # move log file to rundir/jobs
-    jobdir      = os.path.join(DaCycle['dir.da_run'],"jobs")
-    logfile     = 'das.o%s'%DaCycle.DaPlatForm.GetMyID()
-
-    if os.path.exists(logfile):
-        joblogfile  = os.path.join(jobdir,'cycle.%s.log'%DaCycle['time.sample.start'].strftime('%Y%m%d')) 
-        dummy       = shutil.move(logfile,joblogfile)
-        msg         = "....Moved %s to %s"%(logfile,joblogfile)   ; logging.debug(msg)
-
-        msg = "The complete log file is now at: %s"%(joblogfile)   ; logging.info(msg)
-
 
 if __name__ == "__main__":
     pass