Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision
  • classes
  • ctdas-cosmo
  • feature-STILT
  • griddedstatevector
  • ivar
  • karolina
  • liesbeth
  • master
  • package
  • py-3
  • remotec
  • trunk
  • wouter
13 results

Target

Select target project
  • nearrealtimectdas/CTDAS
  • tsurata_a/CTDAS
  • peter050/CTDAS
  • woude033/CTDAS
  • flore019/CTDAS
  • laan035/CTDAS
  • koren007/CTDAS
  • smith036/CTDAS
  • ctdas/CTDAS
9 results
Select Git revision
  • c13_devel
  • coco2
  • ctdas-cosmo
  • ctdas-icon
  • ctdas-wrf
  • ctdas-wrf-in-situ
  • ctdas-wrf-insitu_uoc
  • ctdas-wrf-simple-cmds
  • cte-lagrange
  • ctecc
  • ctecc_europe
  • ctelw_europe
  • ctsf
  • ctsf_sib_mul-add
  • dev-sw-oco2
  • dynamicFFmodel
  • feature-STILT
  • gcp
  • ivar
  • liesbeth
  • master
  • master_cleanup
  • merge_ctecc_master
  • near-real-time
  • new_da_structure
  • old-master
  • old-structure
  • proj-biomass
  • proj-ctecc_hr
  • py-3
  • random_obsop_cte
  • remco
  • remotec
  • tm5mp
  • trunk
35 results
Show changes
Showing
with 1427 additions and 18 deletions
File added
File added
File added
File added
File added
File added
...@@ -70,7 +70,7 @@ class DaSystem(dict): ...@@ -70,7 +70,7 @@ class DaSystem(dict):
""" """
This method loads a DA System Info rc-file with settings for this simulation This method loads a DA System Info rc-file with settings for this simulation
""" """
for k, v in rc.read(rcfilename).items(): for k, v in list(rc.read(rcfilename).items()):
self[k] = v self[k] = v
logging.debug("DA System Info rc-file (%s) loaded successfully" % rcfilename) logging.debug("DA System Info rc-file (%s) loaded successfully" % rcfilename)
...@@ -82,7 +82,7 @@ class DaSystem(dict): ...@@ -82,7 +82,7 @@ class DaSystem(dict):
""" """
needed_rc_items = {} needed_rc_items = {}
for k, v in self.items(): for k, v in list(self.items()):
if v == 'True' : if v == 'True' :
self[k] = True self[k] = True
if v == 'False': if v == 'False':
......
"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters.
Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
updates of the code. See also: http://www.carbontracker.eu.
This program is free software: you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software Foundation,
version 3. This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with this
program. If not, see <http://www.gnu.org/licenses/>."""
#!/usr/bin/env python
# control.py
"""
.. module:: dasystem
.. moduleauthor:: Wouter Peters
Revision History:
File created on 26 Aug 2010.
The DaSystem class is found in the module :mod:`dasystem`, or in a specific implementation under the da/ source tree. It is derived from the standard python :class:`dictionary` object.
It describes the details of the data assimilation system used (i.e., CarbonTracker, or CT Methane, or ....) ::
datadir : /Volumes/Storage/CO2/carbontracker/input/ct08/ ! The directory where input data is found
obs.input.dir : ${datadir}/obsnc/with_fillvalue ! the observation input dir
obs.input.fname : obs_forecast.nc ! the observation input file
ocn.covariance : ${datadir}/oif_p3_era40.dpco2.2000.01.hdf ! the ocean flux covariance file
bio.covariance : ${datadir}/covariance_bio_olson19.nc ! the biosphere flux covariance file
deltaco2.prefix : oif_p3_era40.dpco2 ! the type of ocean product used
regtype : olson19_oif30 ! the ecoregion definitions
nparameters : 240 ! the number of parameters to solve for
random.seed : 4385 ! the random seed for the first cycle
regionsfile : transcom_olson19_oif30.hdf ! the ecoregion defintion mask file
! Info on the sites file used
obs.sites.rc : ${datadir}/sites_and_weights_co2.ct10.rc ! the weights in the covariance matric of each obs
The full baseclass description:
.. autoclass:: da.baseclasses.dasystem.DaSystem
:members:
"""
import logging
import da.tools.rc as rc
################### Begin Class DaSystem ###################
class DaSystem(dict):
"""
Information on the data assimilation system used. This is normally an rc-file with settings.
"""
def __init__(self, rcfilename):
"""
Initialization occurs from passed rc-file name, items in the rc-file will be added
to the dictionary
"""
self.ID = 'CarbonTracker CO2' # the identifier gives the platform name
self.load_rc(rcfilename)
logging.debug("Data Assimilation System initialized: %s" % self.ID)
def load_rc(self, rcfilename):
"""
This method loads a DA System Info rc-file with settings for this simulation
"""
for k, v in rc.read(rcfilename).items():
self[k] = v
logging.debug("DA System Info rc-file (%s) loaded successfully" % rcfilename)
def validate(self):
"""
validate the contents of the rc-file given a dictionary of required keys
"""
needed_rc_items = {}
for k, v in self.items():
if v == 'True' :
self[k] = True
if v == 'False':
self[k] = False
for key in needed_rc_items:
if key not in self:
msg = 'Missing a required value in rc-file : %s' % key
logging.error(msg)
raise IOError(msg)
logging.debug('DA System Info settings have been validated succesfully')
################### End Class DaSystem ###################
if __name__ == "__main__":
pass
...@@ -95,11 +95,11 @@ class Platform(object): ...@@ -95,11 +95,11 @@ class Platform(object):
template += """#$ -hold_jid depends \n""" template += """#$ -hold_jid depends \n"""
# First replace from passed dictionary # First replace from passed dictionary
for k, v in joboptions.items(): for k, v in list(joboptions.items()):
while k in template: while k in template:
template = template.replace(k, v) template = template.replace(k, v)
# Fill remaining values with std_options # Fill remaining values with std_options
for k, v in std_joboptions.items(): for k, v in list(std_joboptions.items()):
while k in template: while k in template:
template = template.replace(k, v) template = template.replace(k, v)
return template return template
......
"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters.
Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
updates of the code. See also: http://www.carbontracker.eu.
This program is free software: you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software Foundation,
version 3. This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with this
program. If not, see <http://www.gnu.org/licenses/>."""
#!/usr/bin/env python
# jobcontrol.py
"""
.. module:: platform
.. moduleauthor:: Wouter Peters
Revision History:
File created on 06 Sep 2010.
The Platform class is found in the module :mod:`platform`, or in a specific implementation under the da/source tree.
The platform object holds attributes and methods that allow job control on each specific platform. This includes methods to create and submit jobs, but also to obtain process and/or job ID's. These are needed to control the flow of
the system on each platform.
Typically, every platform needs specific implementations of this object (through inheritance), and you should refer to your specific Platform object documentation for details (see *da/platform/*).
.. autoclass:: da.baseclasses.platform.Platform
:members:
:inherited-members:
"""
import os
import logging
import subprocess
std_joboptions = {'jobname':'test', 'jobaccount':'co2', 'jobnodes':'nserial 1', 'jobshell':'/bin/sh', 'depends':'', 'jobtime':'01:00:00'}
class Platform(object):
"""
This specifies platform dependent options under generic object calls. A platform object is used to control and submit jobs
"""
def __init__(self):
"""
The init function reports the hard-coded ``Identifier`` and ``Version`` of the Platform. Since each new
computer/user requires their own Platform object modifications, the init function is usually overwritten
in the specific implementation of this class
"""
self.ID = 'iPad' # the identifier gives the plaform name
self.version = '1.0' # the platform version used
logging.debug('%s object initialized' % self.ID)
logging.debug('%s version: %s' % (self.ID, self.version))
def give_blocking_flag(self):
return ""
def give_queue_type(self):
return "foreground"
def get_job_template(self, joboptions={}, block=False):
"""
Returns the job template for a given computing system, and fill it with options from the dictionary provided as argument.
The job template should return the preamble of a job that can be submitted to a queue on your platform,
examples of popular queuing systems are:
- SGE
- MOAB
- XGrid
-
A list of job options can be passed through a dictionary, which are then filled in on the proper line,
an example is for instance passing the dictionary {'account':'co2'} which will be placed
after the ``-A`` flag in a ``qsub`` environment.
An extra option ``block`` has been added that allows the job template to be configured to block the current
job until the submitted job in this template has been completed fully.
"""
template = """## \n""" + \
"""## This is a set of dummy names, to be replaced by values from the dictionary \n""" + \
"""## Please make your own platform specific template with your own keys and place it in a subfolder of the da package.\n """ + \
"""## \n""" + \
""" \n""" + \
"""#$ jobname \n""" + \
"""#$ jobaccount \n""" + \
"""#$ jobnodes \n""" + \
"""#$ jobtime \n""" + \
"""#$ jobshell \n """
if 'depends' in joboptions:
template += """#$ -hold_jid depends \n"""
# First replace from passed dictionary
for k, v in joboptions.items():
while k in template:
template = template.replace(k, v)
# Fill remaining values with std_options
for k, v in std_joboptions.items():
while k in template:
template = template.replace(k, v)
return template
def get_my_id(self):
""" Return the process ID, or job ID of the current process or job"""
return os.getpid()
def write_job(self, jobfile, template, jobid):
"""
This method writes a jobfile to the exec dir and makes it executable (mod 477)
"""
#
# Done, write jobfile
#
f = open(jobfile, 'w')
f.write(template)
f.close()
os.chmod(jobfile, 477)
logging.debug("A job file was created (%s)" % jobfile)
def submit_job(self, jobfile, joblog=None, block=False):
"""
:param jobfile: a string with the filename of a jobfile to run
:param joblog: a string with the filename of a logfile to write run output to
:param block: Boolean specifying whether to submit and continue (F), or submit and wait (T)
:rtype: integer
This method submits a jobfile to the queue, and returns the job ID
"""
cmd = ["sh", jobfile]
logging.info("A new task will be started (%s)" % cmd)
if block:
jobid = subprocess.call(cmd)
else:
jobid = subprocess.Popen(cmd).pid
logging.info('Summary:')
logging.info('job script : %s' % jobfile)
logging.info('job log : %s' % joblog)
logging.info('To manage this process:')
logging.info(' # kill process:')
logging.info(' kill %i\n' % jobid)
def kill_job(self, jobid):
""" This method kills a running job """
def job_stat(self, jobid):
""" This method gets the status of a running job """
output = subprocess.Popen(['qstat', jobid], stdout=subprocess.PIPE).communicate()[0]
logging.info(output)
return output
if __name__ == "__main__":
pass
This diff is collapsed.
This diff is collapsed.
File added
File added
File added
...@@ -46,7 +46,7 @@ class CO2DaSystem(DaSystem): ...@@ -46,7 +46,7 @@ class CO2DaSystem(DaSystem):
'regtype'] 'regtype']
for k, v in self.items(): for k, v in list(self.items()):
if v == 'True' : if v == 'True' :
self[k] = True self[k] = True
if v == 'False': if v == 'False':
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.