Commit 7a42849a authored by brunner's avatar brunner
Browse files

No commit message

No commit message
parent 4997e213
......@@ -179,7 +179,7 @@ class StateVector(object):
nparams = self.gridmap.max()
self.griddict = {}
for r in range(1, int(nparams) + 1):
sel = (self.gridmap.flat == r).nonzero()
sel = np.nonzero(self.gridmap.flat == r)
if len(sel[0]) > 0:
self.griddict[r] = sel
......@@ -190,7 +190,7 @@ class StateVector(object):
self.tcmatrix = np.zeros((self.nparams, 23), 'float')
for r in range(1, self.nparams + 1):
sel = (self.gridmap.flat == r).nonzero()
sel = np.nonzero(self.gridmap.flat == r)
if len(sel[0]) < 1:
continue
else:
......
......@@ -13,29 +13,32 @@
!!! Info for the CarbonTracker data assimilation system
datadir : /Storage/CO2/carbontracker/input/ctdas_2012/
datadir : /store/empa/em05/parsenov/
! For ObsPack
obspack.input.id : obspack_co2_1_GLOBALVIEWplus_v2.1_2016-09-02
obspack.input.dir : ${datadir}/obspacks/${obspack.input.id}
obs.sites.rc : ${obspack.input.dir}/summary/sites_weights_Dec2016.rc
obspack.input.id : obspack
obspack.input.dir : ${datadir}
!/obspack !/${obspack.input.id}
obs.sites.rc : ${obspack.input.dir}/summary/sites_weights_ctdas.rc
! Using a second ObsPack (from 1 January 2016)
obspack.input.id2 : obspack_co2_1_NRT_v3.0_2016-06-06
obspack.input.dir2 : ${datadir}/obspacks/${obspack.input.id2}
obs.sites.rc2 : ${obspack.input.dir2}/summary/sites_weights_Dec2016.rc
ocn.covariance : ${datadir}/oceans/oif/cov_ocean.2000.01.nc
deltaco2.prefix : oif_p3_era40.dpco2
bio.cov.dir : ${datadir}/covariances/gridded_NH/
bio.cov.prefix : cov_ecoregion
regtype : gridded_oif30
nparameters : 9835
random.seed : 4385
regionsfile : ${datadir}/covariances/gridded_NH/griddedNHparameters.nc
random.seed.init: ${datadir}/randomseedinit.pickle
!obspack.input.id2 : obspack_co2_1_NRT_v3.0_2016-06-06
!obspack.input.dir2 : ${datadir}/obspacks/${obspack.input.id2}
!obs.sites.rc2 : ${obspack.input.dir2}/summary/sites_weights_Dec2016.rc
!ocn.covariance : ${datadir}/oceans/oif/cov_ocean.2000.01.nc
!deltaco2.prefix : oif_p3_era40.dpco2
!bio.cov.dir : ${datadir}/covariances/gridded_NH/
!bio.cov.prefix : cov_ecoregion
!regtype : gridded_oif30
nparameters : 11
!random.seed : 4385
regionsfile : /store/empa/em05/parsenov/CTDAS/ctdas-py3/da/analysis/regions_ok.nc
extendedregionsfile : /store/empa/em05/parsenov/CTDAS/ctdas-py3/da/analysis/regions_ok.nc
!regionsfile : ${datadir}/covariances/gridded_NH/griddedNHparameters.nc
!random.seed.init: ${datadir}/randomseedinit.pickle
! Include a naming scheme for the variables
......
......@@ -332,7 +332,7 @@ class CycleControl(dict):
strippedname = os.path.split(self['jobrcfilename'])[-1]
self['jobrcfilename'] = os.path.join(self['dir.exec'], strippedname)
shutil.copy(os.path.join(self.dasystem['regionsfile']),os.path.join(self['dir.exec'],'da','analysis','copied_regions.nc'))
# shutil.copy(os.path.join(self.dasystem['regionsfile']),os.path.join(self['dir.exec'],'da','analysis','copied_regions.nc'))
logging.info('Copied regions file to the analysis directory: %s'%os.path.join(self.dasystem['regionsfile']))
if self.dasystem.has_key('extendedregionsfile'):
shutil.copy(os.path.join(self.dasystem['extendedregionsfile']),os.path.join(self['dir.exec'],'da','analysis','copied_regions_extended.nc'))
......
......@@ -8,7 +8,7 @@
echo "All output piped to file template.log"
source /usr/local/Modules/3.2.8/init/sh
source /opt/intel/bin/ifortvars.sh intel64
export HOST='capegrim'
export HOST='daint'
module load python
export icycle_in_job=999
python template.py rc=template.rc -v $1 >& template.log &
......@@ -25,17 +25,16 @@ sys.path.append(os.getcwd())
# Next, import the tools needed to initialize a data assimilation cycle
#################################################################################################
from da.tools.initexit import start_logger, validate_opts_args, parse_options, CycleControl
from da.tools.pipeline import ensemble_smoother_pipeline, header, footer, analysis_pipeline, archive_pipeline
from da.platform.cartesius import CartesiusPlatform
from da.carbondioxide.dasystem import CO2DaSystem
from da.carbondioxide.optimizer import CO2Optimizer
from da.carbondioxide.obspack_geocarbon import ObsPackObservations
#from da.carbondioxide.statevector import CO2StateVector
from da.co2gridded.statevector import CO2GriddedStateVector
#from da.carbondioxide.obspack import ObsPackObservations
#from da.carbondioxide.obs import CO2Observations
from da.tm5.observationoperator import TM5ObservationOperator
from da.tools.initexit import start_logger, validate_opts_args, parse_options, CycleControl
from da.tools.pipeline import ensemble_smoother_pipeline, header, footer
from da.platform.maunaloa import MaunaloaPlatform
from da.baseclasses.dasystem import DaSystem
from da.baseclasses.statevector import StateVector
from da.carbondioxide.obspack_globalviewplus2 import ObsPackObservations
from da.baseclasses.optimizer import Optimizer
from da.baseclasses.observationoperator import ObservationOperator
#from da.analysis.expand_fluxes import save_weekly_avg_1x1_data, save_weekly_avg_state_data, save_weekly_avg_tc_data, save_weekly_avg_ext_tc_data
#from da.analysis.expand_molefractions import write_mole_fractions
#################################################################################################
......@@ -43,9 +42,6 @@ from da.tm5.observationoperator import TM5ObservationOperator
#################################################################################################
start_logger()
logging.info(header + "CTDAS Copyright (C) 2017, Wouter Peters " + footer)
logging.info(header + "This program comes with ABSOLUTELY NO WARRANTY." + footer)
logging.info(header + "This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 3." + footer)
opts, args = parse_options()
opts, args = validate_opts_args(opts, args)
......@@ -55,26 +51,19 @@ opts, args = validate_opts_args(opts, args)
dacycle = CycleControl(opts, args)
###########################################################################################
### IMPORT THE APPLICATION SPECIFIC MODULES HERE, TO BE PASSED INTO THE MAIN PIPELINE!!! ##
###########################################################################################
platform = CartesiusPlatform()
dasystem = CO2DaSystem(dacycle['da.system.rc'])
obsoperator = TM5ObservationOperator(dacycle['da.obsoperator.rc'])
samples = ObsPackObservations()
#samples = CtObservations()
statevector = CO2GriddedStateVector()
#statevector = CO2StateVector()
optimizer = CO2Optimizer()
platform = MaunaloaPlatform()
dasystem = DaSystem(dacycle['da.system.rc'])
obsoperator = ObservationOperator(dacycle['da.obsoperator.rc'])
samples = ObsPackObservations()
statevector = StateVector()
optimizer = Optimizer()
##########################################################################################
################### ENTER THE PIPELINE WITH THE OBJECTS PASSED BY THE USER ###############
##########################################################################################
logging.info(header + "Entering Pipeline " + footer)
ensemble_smoother_pipeline(dacycle, platform, dasystem, samples, statevector, obsoperator, optimizer)
......@@ -84,9 +73,14 @@ ensemble_smoother_pipeline(dacycle, platform, dasystem, samples, statevector, ob
################### All done, extra stuff can be added next, such as analysis
##########################################################################################
analysis_pipeline(dacycle, platform, dasystem, samples, statevector )
logging.info(header + "Starting analysis" + footer)
sys.exit(0)
archive_pipeline(dacycle, platform, dasystem)
save_weekly_avg_1x1_data(dacycle, statevector)
save_weekly_avg_state_data(dacycle, statevector)
save_weekly_avg_tc_data(dacycle, statevector)
save_weekly_avg_ext_tc_data(dacycle)
write_mole_fractions(dacycle)
sys.exit(0)
......
......@@ -28,8 +28,8 @@
!
! The time for which to start and end the data assimilation experiment in format YYYY-MM-DD HH:MM:SS
time.start : 2000-01-01 00:00:00
time.finish : 2000-01-15 00:00:00
time.start : 2012-01-01 00:00:00
time.finish : 2012-01-15 00:00:00
! Whether to restart the CTDAS system from a previous cycle, or to start the sequence fresh. Valid entries are T/F/True/False/TRUE/FALSE
......@@ -37,29 +37,29 @@ time.restart : False
! The length of a cycle is given in days, such that the integer 7 denotes the typically used weekly cycle. Valid entries are integers > 1
time.cycle : 7
time.cycle : 1
! The number of cycles of lag to use for a smoother version of CTDAS. CarbonTracker CO2 typically uses 5 weeks of lag. Valid entries are integers > 0
time.nlag : 5
time.nlag : 3
! The directory under which the code, input, and output will be stored. This is the base directory for a run. The word
! '/' will be replaced through the start_ctdas.sh script by a user-specified folder name. DO NOT REPLACE
dir.da_run : template
dir.da_run : /scratch/snx3000/parsenov/projname
! The resources used to complete the data assimilation experiment. This depends on your computing platform.
! The number of cycles per job denotes how many cycles should be completed before starting a new process or job, this
! allows you to complete many cycles before resubmitting a job to the queue and having to wait again for resources.
! Valid entries are integers > 0
da.resources.ncycles_per_job : 2
da.resources.ncycles_per_job : 1
! The ntasks specifies the number of threads to use for the MPI part of the code, if relevant. Note that the CTDAS code
! itself is not parallelized and the python code underlying CTDAS does not use multiple processors. The chosen observation
! operator though might use many processors, like TM5. Valid entries are integers > 0
da.resources.ntasks : 30
da.resources.ntasks : 1
! This specifies the amount of wall-clock time to request for each job. Its value depends on your computing platform and might take
! any form appropriate for your system. Typically, HPC queueing systems allow you a certain number of hours of usage before
......@@ -77,11 +77,11 @@ da.system : CarbonTracker
! The specific settings for your system are read from a separate rc-file, which points to the data directories, observations, etc
da.system.rc : da/rc/carbontracker.rc
da.system.rc : da/rc/carbontracker_random.rc
! This flag should probably be moved to the da.system.rc file. It denotes which type of filtering to use in the optimizer
da.system.localization : CT2007
da.system.localization : None
! Info on the observation operator to be used, these keys help to identify the settings for the transport model in this case
......@@ -99,7 +99,7 @@ da.obsoperator.rc : ${da.obsoperator.home}/tm5-ctdas-ei-zoom.rc
! The number of ensemble members used in the experiment. Valid entries are integers > 2
!
da.optimizer.nmembers : 150
da.optimizer.nmembers : 10
! Finally, info on the archive task, if any. Archive tasks are run after each cycle to ensure that the results of each cycle are
! preserved, even if you run on scratch space or a temporary disk. Since an experiment can take multiple weeks to complete, moving
......@@ -121,7 +121,7 @@ da.optimizer.nmembers : 150
!task.rsync.alldata.sourcedirs : ${dir.da_run}
!task.rsync.alldata.destinationdir : you@yourserver.com:/yourfolder/
!task.rsync.alldata.flags : -auv -e ssh
!task.rsync.alldata.flags g -auv -e ssh
! Repeated for rsync task 2, note that we only back up the analysis and output dirs here
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment