Commit 59aa6a46 authored by Peters, Wouter's avatar Peters, Wouter
Browse files

next effort on writing documentation started

parent 4ecd6644
......@@ -7,6 +7,30 @@ Author : peters
Revision History:
File created on 26 Aug 2010.
The DaSystem class is found in the module :mod:`da.baseclasses.dasystem`, or in a specific implementation under the da/ source tree. It is derived from the standard python :class:`dictionary` object.
It describes the details of the data assimilation system used (i.e., CarbonTracker, or CT Methane, or ....).
An example rc-file for CarbonTracker is:::
!!! Info for the CarbonTracker data assimilation system
datadir : /data/CO2/carbontracker/ct08/
obs.input.dir : ${datadir}/obsnc/with_fillvalue/
obs.input.fname : obs_final.nc
ocn.covariance : ${datadir}/oif_p3_era40.dpco2.2000.01.hdf
bio.covariance : ${datadir}/covariance_bio_olson19.nc
deltaco2.prefix : oif_p3_era40.dpco2
regtype : olson19_oif30
nparameters : 240
random.seed : 4385
regionsfile : transcom_olson19_oif30.hdf
The full baseclass description:
.. autoclass:: da.baseclasses.dasystem.DaSystem
:members:
"""
import os
......
......@@ -7,11 +7,18 @@ Author : peters
Revision History:
File created on 28 Jul 2010.
.. autoclass:: da.baseclasses.obs.Observation
:members: Initialize, Validate, AddObs, AddSimulations, AddModelDataMismatch, WriteSampleInfo
.. autoclass:: da.baseclasses.obs.ObservationList
:members: __init__
"""
import os
import sys
import logging
import datetime
from numpy import array, ndarray
identifier = 'Observation baseclass'
version = '0.0'
......@@ -19,9 +26,30 @@ version = '0.0'
################### Begin Class Observation ###################
class Observation(object):
""" an object that holds data + methods and attributes needed to manipulate observations for a DA cycle """
"""
The baseclass Observation is a generic object that provides a number of methods required for any type of observations used in
a data assimilation system. These methods are called from the CarbonTracker pipeline.
.. note:: Most of the actual functionality will need to be provided through a derived Observations class with the methods
below overwritten. Writing your own derived class for Observations is one of the first tasks you'll likely
perform when extending or modifying the CarbonTracker Data Assimilation Shell.
Upon initialization of the class, an object is created that holds no actual data, but has a placeholder attribute `self.Data`
which is an empty list of type :class:`~da.baseclasses.obs.ObservationList`. An ObservationList object is created when the
method :meth:`~da.baseclasses.obs.Observation.AddObs` is invoked in the pipeline.
From the list of observations, a file is written by method
:meth:`~da.baseclasses.obs.Observation.WriteSampleInfo`
with the sample info needed by the
:class:`~da.baseclasses.observationoperator.ObservationOperator` object. The values returned after sampling
are finally added by :meth:`~da.baseclasses.obs.Observation.AddSimulations`
"""
def __init__(self):
"""
create an object with an identifier, version, and an empty ObservationList
"""
self.Identifier = self.getid()
self.Version = self.getversion()
self.Data = ObservationList([]) # Initialize with an empty list of obs
......@@ -29,9 +57,15 @@ class Observation(object):
msg = 'Observation object initialized: %s'%self.Identifier ; logging.info(msg)
def getid(self):
"""
Return the identifier of the Observation Object, used for logging purposed
"""
return identifier
def getversion(self):
"""
Return the version of the Observation Object, used for logging purposed
"""
return identifier
def __str__(self):
......@@ -49,8 +83,12 @@ class Observation(object):
"""
def AddObs(self):
""" Add the observation data to the Observation object. This is in a form of a list that goes under the name Data. The
list has as only requirement that it can return the observed+simulated values through a method "getvalues"
"""
Add actual observation data to the Observation object. This is in a form of an
:class:`~da.baseclasses.obs.ObservationList` that is contained in self.Data. The
list has as only requirement that it can return the observed+simulated values
through the method :meth:`~da.baseclasses.obs.ObservationList.getvalues`
"""
def AddSimulations(self):
......@@ -76,15 +114,14 @@ class Observation(object):
class ObservationList(list):
""" This is a special type of list that holds observed sample objects. It has methods to extract data from such a list easily """
from numpy import array, ndarray
def getvalues(self,name,constructor=array):
from numpy import ndarray
result = constructor([getattr(o,name) for o in self])
if isinstance(result,ndarray):
return result.squeeze()
else:
return result
################### End Class MixingRatioList ###################
result = constructor([getattr(o,name) for o in self])
if isinstance(result,ndarray):
return result.squeeze()
else:
return result
################### End Class ObservationList ###################
......@@ -7,6 +7,17 @@ Author : peters
Revision History:
File created on 06 Sep 2010.
The PlatForm class is found in the module :mod:`da.baseclasses.platform`, or in a specific implementation under the da/source tree.
The platform object holds attributes and methods that allow job control on each specific platform. This includes methods to create and submit jobs, but also to obtain process and/or job ID's. These are needed to control the flow of
the system on each platform.
Typically, every platform needs specific implementations of this object (through inheritance), and you should refer to your specific PlatForm object documentation for details (see *da/platform/*).
.. autoclass:: da.baseclasses.platform.PlatForm
:members:
:inherited-members:
"""
import sys
......
......@@ -3,7 +3,7 @@
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
SPHINXBUILD = sphinx-build-2.6
PAPER =
BUILDDIR = build
......
......@@ -2,42 +2,4 @@
Cycle Control
======================
The CycleControl class is found in the module :mod:`da.tools.initexit`. It is derived from the standard python :class:`dictionary` object. It is the only core object of CTDAS that is automatically created in the pipeline, the user (normally) does not need to modify or extend it. The class is created based on options and arguments passes on the command line when submitting your main CTDAS job.
Valid options are defined in
.. autofunction:: da.tools.initexit.ParseOptions
With the name of a valid ``rc-file``, the CycleControl object is instantiated and validated. An example rc-file looks
like this:::
! Info on the data assimilation cycle
time.restart : False
time.start : 2000-01-01 00:00:00
time.finish : 2000-01-08 00:00:00
time.cycle : 7
time.nlag : 2
dir.da_run : ${HOME}/tmp/test_da
! Info on the DA system used
da.system : CarbonTracker
da.system.rc : carbontracker.rc
da.platform : maunaloa
! Info on the forward model to be used
forecast.model : TM5
forecast.model.rc : ${HOME}/Modeling/TM5/ct_new.rc
forecast.nmembers : 2
The most important method of the CycleControl object are listed below:
.. autoclass:: da.tools.initexit.CycleControl
:members: Initialize, Finalize, SubmitNextCycle, CollectSaveData, MoveSaveData, SetupFileStructure, RecoverRun, RandomSeed
Two important attributes of the CycleControl object are:
(1) DaSystem, an instance of a :ref:`dasystem`
(2) DaPlatForm, an instance of a :ref:`platform`
.. automodule:: da.tools.initexit
......@@ -2,27 +2,4 @@
DaSystem
======================
The DaSystem class is found in the module :mod:`da.baseclasses.dasystem`, or in a specific implementation under the da/ source tree. It is derived from the standard python :class:`dictionary` object.
It describes the details of the data assimilation system used (i.e., CarbonTracker, or CT Methane, or ....).
An example rc-file for CarbonTracker is:::
!!! Info for the CarbonTracker data assimilation system
datadir : /data/CO2/carbontracker/ct08/
obs.input.dir : ${datadir}/obsnc/with_fillvalue/
obs.input.fname : obs_final.nc
ocn.covariance : ${datadir}/oif_p3_era40.dpco2.2000.01.hdf
bio.covariance : ${datadir}/covariance_bio_olson19.nc
deltaco2.prefix : oif_p3_era40.dpco2
regtype : olson19_oif30
nparameters : 240
random.seed : 4385
regionsfile : transcom_olson19_oif30.hdf
The full baseclass description:
.. autoclass:: da.baseclasses.dasystem.DaSystem
:members: __init__
.. automodule:: da.baseclasses.dasystem
......@@ -10,4 +10,8 @@ Documentation
The CycleControl Object <cyclecontrol>
The DASystem Object <dasystem>
The PlatForm Object <platform>
The Observations Object <observations>
The CarbonTracker Observations Object <observations>
The ObservationOperator Object <observationoperator>
The StateVector Object <statevector>
The Optimizer Object <optimizer>
......@@ -2,15 +2,5 @@
PlatForm
======================
The PlatForm class is found in the module :mod:`da.baseclasses.platform`, or in a specific implementation under the da/source tree.
The platform object holds attributes and methods that allow job control on each specific platform. This includes methods to create and submit jobs, but also to obtain process and/or job ID's. These are needed to control the flow of
the system on each platform.
Typically, every platform needs specific implementations of this object (through inheritance), and you should refer to your specific PlatForm object documentation for details (see *da/platform/*).
We give here the class description of the baseclass platform:
.. autoclass:: da.baseclasses.platform.PlatForm
:members:
:inherited-members:
.. automodule:: da.baseclasses.platform
......@@ -7,6 +7,51 @@ Author : peters
Revision History:
File created on 13 May 2009.
The CycleControl class is found in the module :mod:`da.tools.initexit`. It is derived from the standard python :class:`dictionary` object. It is the only core object of CTDAS that is automatically created in the pipeline, the user (normally) does not need to modify or extend it. The class is created based on options and arguments passes on the command line when submitting your main CTDAS job.
Valid options are defined in
.. autofunction:: da.tools.initexit.ParseOptions
With the name of a valid ``rc-file``, the CycleControl object is instantiated and validated. An example rc-file looks
like this:::
! Info on the data assimilation cycle
time.restart : False
time.start : 2000-01-01 00:00:00
time.finish : 2000-01-08 00:00:00
time.cycle : 7
time.nlag : 2
dir.da_run : ${HOME}/tmp/test_da
! Info on the DA system used
da.system : CarbonTracker
da.system.rc : carbontracker.rc
da.platform : maunaloa
! Info on the forward model to be used
forecast.model : TM5
forecast.model.rc : ${HOME}/Modeling/TM5/ct_new.rc
forecast.nmembers : 2
The most important method of the CycleControl object are listed below:
.. autoclass:: da.tools.initexit.CycleControl
:members: Initialize, Finalize, SubmitNextCycle, CleanUpCycle, SetupFileStructure, RecoverRun, RandomSeed
Two important attributes of the CycleControl object are:
(1) DaSystem, an instance of a :ref:`dasystem`
(2) DaPlatForm, an instance of a :ref:`platform`
Other functions in the module initexit that are related to the control of a DA cycle are:
.. autofunction:: da.tools.initexit.StartLogger
.. autofunction:: da.tools.initexit.ValidateOptsArgs
"""
needed_da_items=[
......@@ -543,6 +588,23 @@ class CycleControl(dict):
return None
def CleanUpCycle(self):
"""
Move the log file, clean up rundir after a cycle has finished
"""
# move log file to rundir/jobs
jobdir = os.path.join(self['dir.da_run'],"jobs")
logfile = 'das.o%s'%self.DaPlatForm.GetMyID()
if os.path.exists(logfile):
joblogfile = os.path.join(jobdir,'cycle.%s.log'%self['time.sample.start'].strftime('%Y%m%d'))
dummy = shutil.move(logfile,joblogfile)
msg = "....Moved %s to %s"%(logfile,joblogfile) ; logging.debug(msg)
msg = "The complete log file is now at: %s"%(joblogfile) ; logging.info(msg)
def StartLogger():
""" start the logging of messages to screen"""
......@@ -636,22 +698,6 @@ def ValidateOptsArgs(opts,args):
return opts,args
def CleanUpCycle(DaCycle):
"""
Move files, clean up rundir after a cycle has finished
"""
# move log file to rundir/jobs
jobdir = os.path.join(DaCycle['dir.da_run'],"jobs")
logfile = 'das.o%s'%DaCycle.DaPlatForm.GetMyID()
if os.path.exists(logfile):
joblogfile = os.path.join(jobdir,'cycle.%s.log'%DaCycle['time.sample.start'].strftime('%Y%m%d'))
dummy = shutil.move(logfile,joblogfile)
msg = "....Moved %s to %s"%(logfile,joblogfile) ; logging.debug(msg)
msg = "The complete log file is now at: %s"%(joblogfile) ; logging.info(msg)
if __name__ == "__main__":
pass
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment