.gitignore is now working

da/archive/methane/analysis/tools_regions.py.bak

-"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters. 
-Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
-updates of the code. See also: http://www.carbontracker.eu. 
-
-This program is free software: you can redistribute it and/or modify it under the
-terms of the GNU General Public License as published by the Free Software Foundation, 
-version 3. This program is distributed in the hope that it will be useful, but 
-WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
-FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. 
-
-You should have received a copy of the GNU General Public License along with this 
-program. If not, see <http://www.gnu.org/licenses/>."""
-#!/usr/bin/env python
-
-import numpy as np
-import cPickle
-
-def state_to_grid(values, regionmap, reverse=False, avg=False, mapname=None):
-    """ 
-    This method converts parameters from a CarbonTracker StateVector object to a gridded map of linear multiplication values. These
-    can subsequently be used in the transport model code to multiply/manipulate fluxes
-
-    """
-    nregions = regionmap.max()
-    try:
-        if not mapname: 
-            raise Exception 
-
-        regionselect = cPickle.load(open('%s_regiondict.pickle' % mapname, 'rb'))
-    except:
-
-        # dictionary for region <-> map conversions
-        regs = {}
-        for r in np.arange(1, nregions + 1):
-            sel = (regionmap.flat == r).nonzero()
-            if len(sel[0]) > 0: 
-                regs[r] = sel
-
-        regionselect = regs
-        
-        cPickle.dump(regionselect, open('%s_regiondict.pickle' % mapname, 'wb'), -1)
-        print 'Pickling region map'
-
-    if reverse:
-        """ project 1x1 degree map onto ecoregions """
-
-        result = np.zeros(nregions, float)
-        for k, v in regionselect.iteritems():
-            if avg: 
-                result[k - 1] = values.ravel().take(v).mean()
-            else : 
-                result[k - 1] = values.ravel().take(v).sum()
-        return result
-
-    else:
-        """ project ecoregion properties onto 1x1 degree map """
-
-        result = np.zeros((180, 360,), float)
-        for k, v in regionselect.iteritems():
-            result.put(v, values[k - 1])
-
-        return result
-
-def globarea(im=360, jm=180, silent=True):
-    """ Function calculates the surface area according to TM5 definitions"""
-
-    radius = 6.371e6  # the earth radius in meters
-    deg2rad = np.pi / 180.
-    g = 9.80665 
-
-    dxx = 360.0 / im * deg2rad 
-    dyy = 180.0 / jm * deg2rad 
-    lat = np.arange(-90 * deg2rad, 90 * deg2rad, dyy)
-    dxy = dxx * (np.sin(lat + dyy) - np.sin(lat)) * radius ** 2
-    area = np.resize(np.repeat(dxy, im, axis=0) , [jm, im])
-    if not silent:
-        print 'total area of field = ', np.sum(area.flat)
-        print 'total earth area    = ', 4 * np.pi * radius ** 2
-    return area
-

da/doc/source/conf.py.bak

-"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters. 
-Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
-updates of the code. See also: http://www.carbontracker.eu. 
-
-This program is free software: you can redistribute it and/or modify it under the
-terms of the GNU General Public License as published by the Free Software Foundation, 
-version 3. This program is distributed in the hope that it will be useful, but 
-WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
-FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. 
-
-You should have received a copy of the GNU General Public License along with this 
-program. If not, see <http://www.gnu.org/licenses/>."""
-# -*- coding: utf-8 -*-
-#
-# CarbonTracker Data Assimilation Shell documentation build configuration file, created by
-# sphinx-quickstart on Sun Sep 26 13:39:23 2010.
-#
-# This file is execfile()d with the current directory set to its containing dir.
-#
-# Note that not all possible configuration values are present in this
-# autogenerated file.
-#
-# All configuration values have a default; values that are commented out
-# serve to show the default.
-
-import sys, os
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-sys.path.insert(0, os.path.abspath(os.path.join(os.getcwd(),'../../../')))
-print sys.path
-
-# -- General configuration -----------------------------------------------------
-
-# If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
-
-# Add any Sphinx extension module names here, as strings. They can be extensions
-# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo']
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
-
-# The suffix of source filenames.
-source_suffix = '.rst'
-
-# The encoding of source files.
-#source_encoding = 'utf-8-sig'
-
-# The master toctree document.
-master_doc = 'index'
-
-# General information about the project.
-project = u'CarbonTracker Data Assimilation Shell'
-copyright = u'2010, Wouter Peters'
-
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# The short X.Y version.
-version = '0.1'
-# The full version, including alpha/beta/rc tags.
-release = '0.1'
-
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#language = None
-
-# There are two options for replacing |today|: either, you set today to some
-# non-false value, then it is used:
-#today = ''
-# Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-exclude_patterns = []
-
-# The reST default role (used for this markup: `text`) to use for all documents.
-#default_role = None
-
-# If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
-
-# If true, the current module name will be prepended to all description
-# unit titles (such as .. function::).
-#add_module_names = True
-
-# If true, sectionauthor and moduleauthor directives will be shown in the
-# output. They are ignored by default.
-#show_authors = False
-
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
-
-# A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
-
-
-# -- Options for HTML output ---------------------------------------------------
-
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-html_theme = 'sphinxdoc'
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-#html_theme_options = {}
-
-# Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
-
-# The name for this set of Sphinx documents.  If None, it defaults to
-# "<project> v<release> documentation".
-html_title = 'CarbonTracker DAS'
-
-# A shorter title for the navigation bar.  Default is the same as html_title.
-#html_short_title = None
-
-# The name of an image file (relative to this directory) to place at the top
-# of the sidebar.
-#html_logo = './images/carbontracker.png'
-
-# The name of an image file (within the static path) to use as favicon of the
-# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
-# pixels large.
-#html_favicon = None
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
-
-# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
-# using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
-
-# If true, SmartyPants will be used to convert quotes and dashes to
-# typographically correct entities.
-#html_use_smartypants = True
-
-# Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
-
-# Additional templates that should be rendered to pages, maps page names to
-# template names.
-#html_additional_pages = {}
-
-# If false, no module index is generated.
-#html_domain_indices = True
-
-# If false, no index is generated.
-#html_use_index = True
-
-# If true, the index is split into individual pages for each letter.
-#html_split_index = False
-
-# If true, links to the reST sources are added to the pages.
-#html_show_sourcelink = True
-
-# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#html_show_sphinx = True
-
-# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
-
-# If true, an OpenSearch description file will be output, and all pages will
-# contain a <link> tag referring to it.  The value of this option must be the
-# base URL from which the finished HTML is served.
-#html_use_opensearch = ''
-
-# This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = 'CarbonTrackerDataAssimilationShelldoc'
-
-
-# -- Options for LaTeX output --------------------------------------------------
-
-# The paper size ('letter' or 'a4').
-#latex_paper_size = 'letter'
-
-# The font size ('10pt', '11pt' or '12pt').
-#latex_font_size = '10pt'
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title, author, documentclass [howto/manual]).
-latex_documents = [
-  ('index', 'CarbonTrackerDataAssimilationShell.tex', u'CarbonTracker Data Assimilation Shell Documentation',
-   u'Wouter Peters', 'manual'),
-]
-
-# The name of an image file (relative to this directory) to place at the top of
-# the title page.
-#latex_logo = None
-
-# For "manual" documents, if this is true, then toplevel headings are parts,
-# not chapters.
-#latex_use_parts = False
-
-# If true, show page references after internal links.
-#latex_show_pagerefs = False
-
-# If true, show URL addresses after external links.
-#latex_show_urls = False
-
-# Additional stuff for the LaTeX preamble.
-#latex_preamble = ''
-
-# Documents to append as an appendix to all manuals.
-#latex_appendices = []
-
-# If false, no module index is generated.
-#latex_domain_indices = True
-
-
-# -- Options for manual page output --------------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
-man_pages = [
-    ('index', 'carbontrackerdataassimilationshell', u'CarbonTracker Data Assimilation Shell Documentation',
-     [u'Wouter Peters'], 1)
-]

templates/template.jb

-#! /bin/env bash
-#SBATCH -p short
-#SBATCH -t 01:00:00 
-#SBATCH -n 48
-
-echo "All output piped to file template.log"
-export HOST='cartesius'
-export icycle_in_job=999
-
-.  /projects/0/tm5meteo/admin/bash_profile__2019
-
-module load Python/3.6.6-foss-2018b
-module load pre2019
-module load hdf5/impi/intel/1.8.9
-module load hdf4/intel/4.2.9
-module load netcdf/impi/intel/4.1.3
-module load szip/intel/2.1
-module load fortran
-module load mpi
-module load nco
-
-python3 template.py rc=template.rc -v $1 >& template.log 

templates/template.py

-"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters. 
-Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
-updates of the code. See also: http://www.carbontracker.eu. 
-
-This program is free software: you can redistribute it and/or modify it under the
-terms of the GNU General Public License as published by the Free Software Foundation, 
-version 3. This program is distributed in the hope that it will be useful, but 
-WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
-FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. 
-
-You should have received a copy of the GNU General Public License along with this 
-program. If not, see <http://www.gnu.org/licenses/>."""
-#!/usr/bin/env python
-
-#################################################################################################
-# First order of business is always to make all other python modules accessible through the path
-#################################################################################################
-
-import sys
-import os
-import logging
-sys.path.append(os.getcwd())
-
-#################################################################################################
-# Next, import the tools needed to initialize a data assimilation cycle
-#################################################################################################
-
-from da.tools.initexit import start_logger, validate_opts_args, parse_options, CycleControl
-from da.pipelines.pipeline_cteco2 import ensemble_smoother_pipeline, header, footer, analysis_pipeline, archive_pipeline
-from da.dasystems.dasystem_cteco2 import CO2DaSystem
-from da.platform.cartesius import CartesiusPlatform
-from da.statevectors.statevectorNHgridded_cteco2 import CO2GriddedStateVector 
-from da.observations.obspack_globalviewplus_2020 import ObsPackObservations
-from da.observations.column_obs import TotalColumnObservations
-from da.obsoperators.observationoperator_tm5_cteco2 import TM5ObservationOperator 
-from da.optimizers.optimizer_cteco2 import CO2Optimizer
-
-
-#################################################################################################
-# Parse and validate the command line options, start logging
-#################################################################################################
-
-start_logger()
-opts, args = parse_options()
-opts, args = validate_opts_args(opts, args)
-
-#################################################################################################
-# Create the Cycle Control object for this job    
-#################################################################################################
-
-dacycle = CycleControl(opts, args)
-
-platform    = CartesiusPlatform()
-dasystem    = CO2DaSystem(dacycle['da.system.rc'])
-obsoperator = TM5ObservationOperator(dacycle['da.obsoperator.rc'])
-samples     = [ObsPackObservations(),TotalColumnObservations()]
-statevector = CO2GriddedStateVector()
-optimizer   = CO2Optimizer()
-
-##########################################################################################
-################### ENTER THE PIPELINE WITH THE OBJECTS PASSED BY THE USER ###############
-##########################################################################################
-
-
-logging.info(header + "Entering Pipeline " + footer) 
-
-ensemble_smoother_pipeline(dacycle, platform, dasystem, samples, statevector, obsoperator,optimizer)
-
-
-##########################################################################################
-################### All done, extra stuff can be added next, such as analysis
-##########################################################################################
-
-logging.info(header + "Starting analysis" + footer) 
-
-analysis_pipeline(dacycle, platform, dasystem, samples, statevector )
-
-sys.exit(0)
-
-

templates/template.rc

-! CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters. 
-! Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
-! updates of the code. See also: http://www.carbontracker.eu. 
-!
-! This program is free software: you can redistribute it and/or modify it under the
-! terms of the GNU General Public License as published by the Free Software Foundation, 
-! version 3. This program is distributed in the hope that it will be useful, but 
-! WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
-! FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. 
-!
-! You should have received a copy of the GNU General Public License along with this 
-! program. If not, see <http://www.gnu.org/licenses/>. 
-
-! author: Wouter Peters 
-!
-! This is a blueprint for an rc-file used in CTDAS. Feel free to modify it, and please go to the main webpage for further documentation.
-!
-! Note that rc-files have the convention that commented lines start with an exclamation mark (!), while special lines start with a hashtag (#).
-!
-! When running the script start_ctdas.sh, this /.rc file will be copied to your run directory, and some items will be replaced for you.
-! The result will be a nearly ready-to-go rc-file for your assimilation job. The entries and their meaning are explained by the comments below.
-!
-!
-! HISTORY:
-!
-! Created on August 20th, 2013 by Wouter Peters
-!
-!
-! The time for which to start and end the data assimilation experiment in format YYYY-MM-DD HH:MM:SS
-
-time.start          : 2014-12-27 00:00:00
-time.finish         : 2016-02-01 00:00:00
-
-! Whether to restart the CTDAS system from a previous cycle, or to start the sequence fresh. Valid entries are T/F/True/False/TRUE/FALSE
-
-time.restart        : False
-
-! The length of a cycle is given in days, such that the integer 7 denotes the typically used weekly cycle. Valid entries are integers > 1
-
-time.cycle          : 2
-
-! The number of cycles of lag to use for a smoother version of CTDAS. CarbonTracker CO2 typically uses 5 weeks of lag. Valid entries are integers > 0
-
-time.nlag           : 2
-
-! The directory under which the code, input, and output will be stored. This is the base directory for a run. The word
-! '/' will be replaced through the start_ctdas.sh script by a user-specified folder name. DO NOT REPLACE
-
-dir.da_run          : template
-
-! The resources used to complete the data assimilation experiment. This depends on your computing platform.
-! The number of cycles per job denotes how many cycles should be completed before starting a new process or job, this
-! allows you to complete many cycles before resubmitting a job to the queue and having to wait again for resources.
-! Valid entries are integers > 0
-
-da.resources.ncycles_per_job : 1
-
-! The ntasks specifies the number of threads to use for the MPI part of the code, if relevant. Note that the CTDAS code
-! itself is not parallelized and the python code underlying CTDAS does not use multiple processors. The chosen observation
-! operator though might use many processors, like TM5. Valid entries are integers > 0
-
-da.resources.ntasks : 48
-
-! This specifies the amount of wall-clock time to request for each job. Its value depends on your computing platform and might take
-! any form appropriate for your system. Typically, HPC queueing systems allow you a certain number of hours of usage before 
-! your job is killed, and you are expected to finalize and submit a next job before that time. Valid entries are strings.
-
-da.resources.ntime  : 5:00:00
-
-! The resource settings above will cause the creation of a job file in which 2 cycles will be run, and 30 threads 
-! are asked for a duration of 4 hours
-!
-! Info on the DA system used, this depends on your application of CTDAS and might refer to for instance CO2, or CH4 optimizations.
-!
-
-da.system           : CarbonTracker
-
-! The specific settings for your system are read from a separate rc-file, which points to the data directories, observations, etc
-
-da.system.rc        : da/rc/carbontracker_gcp2020_insitu_OCO2.rc
-
-! This flag should probably be moved to the da.system.rc file. It denotes which type of filtering to use in the optimizer
-
-da.system.localization : None
-
-! Info on the observation operator to be used, these keys help to identify the settings for the transport model in this case
-
-da.obsoperator         : TM5
-
-!
-! The TM5 transport model is controlled by an rc-file as well. The value below refers to the configuration of the TM5 model to 
-! be used as observation operator in this experiment.
-!
-
-da.obsoperator.home    : ${HOME}/TM5
-da.obsoperator.rc      : ${da.obsoperator.home}/tm5-test_py3.rc
-
-!
-! The number of ensemble members used in the experiment. Valid entries are integers > 2
-!
-
-da.optimizer.nmembers  : 150
-
-! Finally, info on the archive task, if any. Archive tasks are run after each cycle to ensure that the results of each cycle are
-! preserved, even if you run on scratch space or a temporary disk. Since an experiment can take multiple weeks to complete, moving
-! your results out of the way, or backing them up, is usually a good idea. Note that the tasks are commented and need to be uncommented
-! to use this feature.
-
-! The following key identifies that two archive tasks will be executed, one called 'alldata' and the other 'resultsonly'. 
-
-!task.rsync : alldata onlyresults
-
-! The specifics for the first task. 
-! 1> Which source directories to back up. Valid entry is a list of folders separated by spaces
-! 2> Which destination directory to use. Valid entries are a folder name, or server and folder name in rsync format as below
-! 3> Which flags to add to the rsync command
-! The settings below will result in an rsync command that looks like:
-!
-!       rsync -auv -e ssh ${dir.da_run} you@yourserver.com:/yourfolder/
-!
-
-!task.rsync.alldata.sourcedirs : ${dir.da_run}
-!task.rsync.alldata.destinationdir : you@yourserver.com:/yourfolder/
-!task.rsync.alldata.flags g -auv -e ssh
-
-! Repeated for rsync task 2, note that we only back up the analysis and output dirs here
-
-!task.rsync.onlyresults.sourcedirs : ${dir.da_run}/analysis ${dir.da_run}/output
-!task.rsync.onlyresults.destinationdir : you@yourserver.com:/yourfolder/
-!task.rsync.onlyresults.flags : -auv -e ssh
-