templates updates with new options, huygens refs removed

template.jb

-## @ node_usage = shared
-# @ notification = never
-# @ input = /dev/null
-# @ output = template.out.$(jobid)
-# @ error = template.out.$(jobid)
-# @ wall_clock_limit = 24:00:00
-# @ job_type = serial 
-# @ shell = /bin/bash 
-# @ queue 
-
+#! /bin/env bash
+#SBATCH -p normal
+#SBATCH -t 12:00:00 
 
 echo ########################
-module load intel/latest/tm5
-module load python/2.6
+module load python
+module load nco
 
 export icycle_in_job=999
-/opt/local/bin/python template.py rc=template.rc $1 >& template.log &
+python template.py rc=template.rc $1 >& template.log &

template.py

@@ -14,8 +14,8 @@ sys.path.append(os.getcwd())
 #################################################################################################
 
 from da.tools.initexit import start_logger, validate_opts_args, parse_options, CycleControl 
-from da.tools.pipeline import ensemble_smoother_pipeline, header, footer
-from da.platform.huygens import HuygensPlatform 
+from da.tools.pipeline import ensemble_smoother_pipeline, header, footer, analysis_pipeline, archive_pipeline
+from da.platform.cartesius import CartesiusPlatform 
 from da.carbondioxide.dasystem import CO2DaSystem 
 from da.carbondioxide.optimizer import CO2Optimizer
 from da.carbondioxide.obspack_geocarbon import ObsPackObservations
@@ -25,10 +25,6 @@ from da.co2gridded.statevector import CO2GriddedStateVector
 #from da.carbondioxide.obs import CO2Observations 
 from da.tm5.observationoperator import TM5ObservationOperator 
 
-from da.analysis.expand_fluxes import save_weekly_avg_1x1_data, save_weekly_avg_state_data, save_weekly_avg_tc_data, save_weekly_avg_ext_tc_data, save_weekly_avg_agg_data
-from da.analysis.expand_molefractions import write_mole_fractions
-from da.analysis.summarize_obs import summarize_obs
-from da.analysis.time_avg_fluxes import time_avg
 
 #################################################################################################
 # Parse and validate the command line options, start logging
@@ -49,7 +45,7 @@ dacycle = CycleControl(opts, args)
 ###########################################################################################
 
 
-platform = HuygensPlatform()
+platform = CartesiusPlatform()
 dasystem = CO2DaSystem(dacycle['da.system.rc'])
 obsoperator = TM5ObservationOperator(dacycle['da.obsoperator.rc'])
 samples = ObsPackObservations()
@@ -72,35 +68,9 @@ ensemble_smoother_pipeline(dacycle, platform, dasystem, samples, statevector, ob
 ################### All done, extra stuff can be added next, such as analysis
 ##########################################################################################
 
-logging.info(header + "Starting analysis" + footer) 
-
-try:
-    save_weekly_avg_1x1_data(dacycle, statevector)
-    save_weekly_avg_state_data(dacycle, statevector)
-except:
-    pass
-try:
-    save_weekly_avg_tc_data(dacycle, statevector)
-    save_weekly_avg_ext_tc_data(dacycle)
-except:
-    pass
-try:
-    save_weekly_avg_agg_data(dacycle,region_aggregate='flux1x1')
-    save_weekly_avg_agg_data(dacycle,region_aggregate='transcom')
-    save_weekly_avg_agg_data(dacycle,region_aggregate='olson')
-except:
-    pass
-try:
-    time_avg(dacycle,'flux1x1')
-    time_avg(dacycle,'transcom')
-    time_avg(dacycle,'olson')
-except:
-    pass
-try:
-    write_mole_fractions(dacycle)
-    summarize_obs(dacycle)
-except:
-    pass
+analysis_pipeline(dacycle, platform, dasystem, samples, statevector, obsoperator )
+
+archive_pipeline(dacycle, platform, dasystem)
 
 sys.exit(0)
 

template.rc

 ! Info on the data assimilation cycle
 
 time.restart        : False
-time.start          : 2005-01-01 00:00:00
-time.finish         : 2011-01-01 00:00:00
-time.cycle          : 7
-time.nlag           : 5
-dir.da_run          : template
-random.seed.init    : ${datadir}/randomseedinit.pickle
+time.start          : 2000-01-01 00:00:00
+time.finish         : 2000-01-03 00:00:00
+time.cycle          : 1
+time.nlag           : 2
+dir.da_run          : /scratch/shared/peters/test_ctdas/exec
+!random.seed.init    : ${datadir}/randomseedinit.pickle
 
 ! info on the resources needed
 
@@ -26,3 +26,9 @@ da.system.localization : CT2007
 da.obsoperator         : TM5
 da.obsoperator.rc      : ${HOME}/TM5/tm5-ctdas-ei-zoom.rc
 da.optimizer.nmembers  : 150
+
+! info on the archive task, if any
+
+task.rsync : True
+task.rsync.destination : peters@maunaloa.wur.nl:/Storage/CO2/peters/
+task.rsync.flags : -auvz -e ssh