Skip to content
GitLab
Commit af76cffd authored by Peters, Wouter's avatar Peters, Wouter
Browse files

two new classes documented

parent 4ea4cb7c
Hide whitespace changes
Inline Side-by-side
da/baseclasses/dasystem.py
View file @ af76cffd
......@@ -17,12 +17,14 @@ import datetime
################### Begin Class DaSystem ###################
class DaSystem(dict):
""" Information on the data assimilation system used. This is normally an rc-file with settings.
"""
Information on the data assimilation system used. This is normally an rc-file with settings.
"""
def __init__(self,rcfilename):
"""
Initialization occurs from passed rc-file name
Initialization occurs from passed rc-file name, items in the rc-file will be added
to the dictionary
"""
self.LoadRc(rcfilename)
......
da/baseclasses/platform.py
View file @ af76cffd
......@@ -17,9 +17,16 @@ import subprocess
std_joboptions={'jobname':'test','jobaccount':'co2','jobnodes':'nserial 1','jobshell':'/bin/sh','depends':'','jobtime':'00:30:00'}
class PlatForm(object):
""" This specifies platform dependent options under generic object calls. A platform object is used to control and submit jobs"""
"""
This specifies platform dependent options under generic object calls. A platform object is used to control and submit jobs
"""
def __init__(self):
"""
The init function reports the hard-coded ``Identifier`` and ``Version`` of the PlatForm. Since each new
computer/user requires their own PlatForm object modifications, the init function is usually overwritten
in the specific implementation of this class
"""
self.Identifier = 'iPad' # the identifier gives the plaform name
self.Version = '1.0' # the platform version used
......@@ -32,7 +39,22 @@ class PlatForm(object):
return None
def GetJobTemplate(self,joboptions={},block=False):
""" Return the job template for a given computing system, and fill it with options from the dictionary provided as argument"""
"""
Returns the job template for a given computing system, and fill it with options from the dictionary provided as argument.
The job template should return the preamble of a job that can be submitted to a queue on your platform,
examples of popular queuing systems are:
- SGE
- MOAB
- XGrid
-
A list of job options can be passed through a dictionary, which are then filled in on the proper line,
an example is for instance passing the dictionary {'account':'co2'} which will be placed
after the ``-A`` flag in a ``qsub`` environment.
An extra option ``block`` has been added that allows the job template to be configured to block the current
job until the submitted job in this template has been completed fully.
"""
template = """## \n"""+ \
"""## This is a set of dummy names, to be replaced by values from the dictionary \n"""+ \
......@@ -61,11 +83,15 @@ class PlatForm(object):
return template
def GetMyID(self):
""" Return the process ID, or job ID of the current process or job"""
return os.getpid()
def WriteJob(self,DaCycle,template, jobid):
""" This method writes a jobfile to the exec dir"""
"""
This method writes a jobfile to the exec dir and makes it executable (mod 477)
"""
#
# Done, write jobfile
#
......@@ -82,7 +108,9 @@ class PlatForm(object):
def SubmitJob(self,jobfile):
""" This method submits a jobfile to the queue, and returns the queue ID """
"""
This method submits a jobfile to the queue, and returns the job ID
"""
cmd = ['sh',jobfile]
msg = "A new task will be started (%s)\n\n\n"%cmd ; logging.debug(msg)
......
da/tools/initexit.py
View file @ af76cffd
......@@ -40,13 +40,13 @@ class CycleControl(dict):
def __init__(self,opts=[],args={}):
"""
The CycleControl object is instantiated with a set of options and arguments.
The list of arguments must contain the name of an existing rc-file.
This rc-file is loaded by method :meth:`~da.tools.initexit.LoadRc` and validated
by :meth:`~da.tools.initexit.ValidateRC`
The CycleControl object is instantiated with a set of options and arguments.
The list of arguments must contain the name of an existing ``rc-file``.
This rc-file is loaded by method :meth:`~da.tools.initexit.CycleControl.LoadRc` and validated
by :meth:`~da.tools.initexit.CycleControl.ValidateRC`
Options for the CycleControl consist of accepted command line flags or arguments
in :func:`~da.tools.initexit.ParseOptions`
Options for the CycleControl consist of accepted command line flags or arguments
in :func:`~da.tools.initexit.CycleControl.ParseOptions`
"""
......@@ -96,7 +96,8 @@ in :func:`~da.tools.initexit.ParseOptions`
def ValidateRC(self):
"""
Validate the contents of the rc-file given a dictionary of required keys. Currently required keys are :attr:`~da.tools.initexit.needed_da_items`
Validate the contents of the rc-file given a dictionary of required keys.
Currently required keys are :attr:`~da.tools.initexit.needed_da_items`
"""
from da.tools.general import ToDatetime
......@@ -165,7 +166,11 @@ Validate the contents of the rc-file given a dictionary of required keys. Curren
return None
def RandomSeed(self,action='read'):
""" Get the randomseed and save it, or read the random seed and set it"""
"""
Get the randomseed and save it, or read the random seed and set it. The seed is currently stored
in a python :mod:`pickle` file, residing in the save directory
"""
import cPickle
import numpy as np
......@@ -201,11 +206,29 @@ Validate the contents of the rc-file given a dictionary of required keys. Curren
The choice that gets executed depends on the presence of
1. the -r option on the command line, this triggers a recover
2. the "time.restart : True" option in the da.rc file
1. the ``-r`` option on the command line, this triggers a recover
2. the ``time.restart : True`` option in the da.rc file
The latter is automatically set if the filter submits the next cycle at the end of the current one,
through method :meth:`SubmitNextCycle`.
through method :meth:`~da.tools.initexit.CycleControl.SubmitNextCycle`.
The specific call tree under each scenario is:
1.
* dummy = :meth:`~da.tools.initexit.CycleControl.SetupFileStructure()` <- Create directory tree
2.
* dummy = :meth:`~da.tools.initexit.CycleControl.SetupFileStructure()`
* dummy = :meth:`~da.tools.initexit.CycleControl.RandomSeed` <- Read the random seed from file
3.
* dummy = :meth:`~da.tools.initexit.CycleControl.SetupFileStructure()`
* dummy = :meth:`~da.tools.initexit.CycleControl.RecoverRun()` <- Recover files from save/one-ago dir, reset ``time.start``
* dummy = :meth:`~da.tools.initexit.CycleControl.RandomSeed`
And is always followed by a call to
* ParseTimes()
* WriteRc('jobfilename')
"""
......@@ -253,23 +276,24 @@ Validate the contents of the rc-file given a dictionary of required keys. Curren
def SetupFileStructure(self):
"""
Create file structure needed for data assimilation system.
In principle this looks like:
${da_rundir}
${da_rundir}/input
${da_rundir}/output
${da_rundir}/exec
${da_rundir}/diagnostics
${da_rundir}/analysis
${da_rundir}/jobs
${da_rundir}/save
${da_rundir}/save/one-ago
${da_rundir}/save/two-ago
Note that the exec dir will actually be a simlink to the directory where
the SIBCASA or model executable lives. This directory is passed through
the da.rc file. The observation input files will be placed in the exec dir,
and the resulting simulated values will be retrieved from there as well.
In principle this looks like:
* ``${da_rundir}``
* ``${da_rundir}/input``
* ``${da_rundir}/output``
* ``${da_rundir}/exec``
* ``${da_rundir}/diagnostics``
* ``${da_rundir}/analysis``
* ``${da_rundir}/jobs``
* ``${da_rundir}/save``
* ``${da_rundir}/save/one-ago``
* ``${da_rundir}/save/two-ago``
Note that the exec dir will actually be a simlink to the directory where
the observation operator executable lives. This directory is passed through
the ``da.rc`` file. The observation input files will be placed in the exec dir,
and the resulting simulated values will be retrieved from there as well.
"""
from da.tools.general import CreateDirs
......@@ -300,12 +324,13 @@ Validate the contents of the rc-file given a dictionary of required keys. Curren
def RecoverRun(self):
"""Prepare a recovery from a crashed run. This consists of:
"""
Prepare a recovery from a crashed run. This consists of:
- copying all data from the save/one-ago folder,
- replacing all rc-file items with those from da_runtime.rc
- resetting the seed of the random number generator to the value it had before the crash
- copying all data from the save/one-ago folder (:meth:`~da.tools.initexit.CycleControl.MoveSaveData`),
- replacing all ``rc-file`` items with those from ``da_runtime.rc``
- resetting the seed of the random number generator to the value it had before the crash (:meth:`~da.tools.initexit.CycleControl.RandomSeed`)
"""
# Move all data from the save/one-ago directory to the save/current directory
......@@ -333,7 +358,7 @@ Validate the contents of the rc-file given a dictionary of required keys. Curren
The following sequence of actions occur:
* Write the randomseed to file for reuse in next cycle
* Write a new rc-file with time.restart : True, and new time.start and time.end
* Write a new ``rc-file`` with ``time.restart : True``, and new ``time.start`` and ``time.end``
* Collect all needed data needed for check-pointing (restart from current system state)
* Move the previous check pointing data out of the way, and replace with current
* Submit the next cycle
......@@ -351,8 +376,8 @@ Validate the contents of the rc-file given a dictionary of required keys. Curren
All files needded are written to the save/ directory.
Currently, the following files are included
- The randomseed.pickle file, also already in the save directory
- The da_runtime.rc file
- The ``randomseed.pickle`` file, also already in the save directory
- The ``da_runtime.rc`` file
- The savestate.yyyymmdd.nc file
- ...
......@@ -379,7 +404,10 @@ Validate the contents of the rc-file given a dictionary of required keys. Curren
def MoveSaveData(self, io_option='restore', save_option='partial',filter=[]):
""" Store or restore model state to/from a temporary directory. Two save_option options are available:
"""
:
Store or restore model state to/from a temporary directory. Two save_option options are available:
(1) save_option = 'partial' : Save the background concentration fields only, to use for a later advance of the background
(2) save_option = 'full' : Save all restart data, to use for a next cycle
......@@ -512,8 +540,14 @@ Validate the contents of the rc-file given a dictionary of required keys. Curren
def SubmitNextCycle(self):
"""
Submit next job in cycle, if needed. The end date is held against the final date. A subprocess is called that
submits the next job, without waiting for its result so this cycle can finish and exit.
Submit the next job of a DA cycle, this consists of
* getting a job template as returned by :meth:`~da.tools.baseclasses.platform.GetJobTemplate`
* adding the lines needed to start a next run with a newly created rc-file
* Writing the jobfile as done by :meth:`~da.tools.baseclasses.platform.WriteJob`
* Submitting the jobfile as done by :meth:`~da.tools.baseclasses.platform.WriteJob`
If the end of the cycle series is reached, no new job is submitted.
"""
import subprocess
import os
......@@ -537,7 +571,14 @@ Validate the contents of the rc-file given a dictionary of required keys. Curren
return None
def SubmitSubStep(self,stepname):
""" submit the next substep of a DA cycle"""
"""
Submit the next substep of a DA cycle, this consists of
* getting a job template as returned by :meth:`~da.tools.baseclasses.platform.GetJobTemplate`
* adding the lines needed to start a next run with a newly created rc-file
* Writing the jobfile as done by :meth:`~da.tools.baseclasses.platform.WriteJob`
* Submitting the jobfile as done by :meth:`~da.tools.baseclasses.platform.WriteJob`
"""
import subprocess
import os
from string import join
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment