Commit e3f2a894 authored by karolina's avatar karolina
Browse files

ordered imports (mainly moved to the top of the module)

parent 4863fc08
...@@ -3,13 +3,23 @@ ...@@ -3,13 +3,23 @@
import sys import sys
sys.path.append('../../') sys.path.append('../../')
import os import os
import getopt
from datetime import datetime, timedelta from datetime import datetime, timedelta
from da.tools.general import create_dirs
from da.analysis.tools_country import countryinfo # needed here
import logging import logging
import numpy as np import numpy as np
from pylab import date2num, num2date from pylab import date2num, num2date
import da.tools.io4 as io
from da.analysis.tools_regions import globarea, state_to_grid
from da.tools.general import create_dirs
from da.analysis.tools_country import countryinfo # needed here
from da.analysis.tools_transcom import transcommask, ExtendedTCRegions
import da.analysis.tools_transcom as tc
import da.analysis.tools_country as ct
import da.analysis.tools_time as timetools
""" """
Author: Wouter Peters (Wouter.Peters@noaa.gov) Author: Wouter Peters (Wouter.Peters@noaa.gov)
...@@ -38,10 +48,6 @@ def save_weekly_avg_1x1_data(DaCycle, StateVector): ...@@ -38,10 +48,6 @@ def save_weekly_avg_1x1_data(DaCycle, StateVector):
:param StateVector: a :class:`~da.baseclasses.statevector.StateVector` :param StateVector: a :class:`~da.baseclasses.statevector.StateVector`
:rtype: None :rtype: None
""" """
import da.tools.io4 as io
from da.analysis.tools_regions import globarea
# #
dirname = create_dirs(os.path.join(DaCycle['dir.analysis'], 'data_flux1x1_weekly')) dirname = create_dirs(os.path.join(DaCycle['dir.analysis'], 'data_flux1x1_weekly'))
# #
...@@ -217,10 +223,6 @@ def save_weekly_avg_state_data(DaCycle, StateVector): ...@@ -217,10 +223,6 @@ def save_weekly_avg_state_data(DaCycle, StateVector):
:param StateVector: a :class:`~da.baseclasses.statevector.StateVector` :param StateVector: a :class:`~da.baseclasses.statevector.StateVector`
:rtype: None :rtype: None
""" """
import da.tools.io4 as io
import logging
from da.analysis.tools_regions import globarea
dirname = create_dirs(os.path.join(DaCycle['dir.analysis'], 'data_state_weekly')) dirname = create_dirs(os.path.join(DaCycle['dir.analysis'], 'data_state_weekly'))
# #
...@@ -452,9 +454,7 @@ def save_weekly_avg_tc_data(DaCycle, StateVector): ...@@ -452,9 +454,7 @@ def save_weekly_avg_tc_data(DaCycle, StateVector):
these with the parameters in the StateVector. This creates posterior fluxes, and the posterior covariance for the complete these with the parameters in the StateVector. This creates posterior fluxes, and the posterior covariance for the complete
StateVector in units of mol/box/s which we then turn into TC fluxes and covariances. StateVector in units of mol/box/s which we then turn into TC fluxes and covariances.
""" """
from da.analysis.tools_regions import globarea
from da.analysis.tools_transcom import transcommask
import da.tools.io4 as io
# #
dirname = create_dirs(os.path.join(DaCycle['dir.analysis'], 'data_tc_weekly')) dirname = create_dirs(os.path.join(DaCycle['dir.analysis'], 'data_tc_weekly'))
# #
...@@ -584,9 +584,7 @@ def save_weekly_avg_ext_tc_data(DaCycle): ...@@ -584,9 +584,7 @@ def save_weekly_avg_ext_tc_data(DaCycle):
*** Example *** *** Example ***
./expand_savestate project=enkf_release sd=20000101 ed=20010101 """ ./expand_savestate project=enkf_release sd=20000101 ed=20010101 """
from da.analysis.tools_transcom import ExtendedTCRegions
import da.tools.io4 as io
import logging
# #
dirname = create_dirs(os.path.join(DaCycle['dir.analysis'], 'data_tc_weekly')) dirname = create_dirs(os.path.join(DaCycle['dir.analysis'], 'data_tc_weekly'))
# #
...@@ -704,11 +702,7 @@ def save_weekly_avg_agg_data(DaCycle, region_aggregate='olson'): ...@@ -704,11 +702,7 @@ def save_weekly_avg_agg_data(DaCycle, region_aggregate='olson'):
these with the parameters in the StateVector. This creates posterior fluxes, and the posterior covariance for the complete these with the parameters in the StateVector. This creates posterior fluxes, and the posterior covariance for the complete
StateVector in units of mol/box/s which we then turn into TC fluxes and covariances. StateVector in units of mol/box/s which we then turn into TC fluxes and covariances.
""" """
from da.analysis.tools_regions import globarea, state_to_grid
import da.analysis.tools_transcom as tc
import da.analysis.tools_country as ct
import da.tools.io4 as io
import logging
# #
dirname = create_dirs(os.path.join(DaCycle['dir.analysis'], 'data_%s_weekly' % region_aggregate)) dirname = create_dirs(os.path.join(DaCycle['dir.analysis'], 'data_%s_weekly' % region_aggregate))
# #
...@@ -918,9 +912,6 @@ def save_time_avg_data(DaCycle, infile, avg='monthly'): ...@@ -918,9 +912,6 @@ def save_time_avg_data(DaCycle, infile, avg='monthly'):
*** Example *** *** Example ***
./expand_savestate project=enkf_release sd=20000101 ed=20010101 """ ./expand_savestate project=enkf_release sd=20000101 ed=20010101 """
import da.analysis.tools_time as timetools
import da.tools.io4 as io
if 'weekly' in infile: if 'weekly' in infile:
intime = 'weekly' intime = 'weekly'
if 'monthly' in infile: if 'monthly' in infile:
......
#!/usr/bin/env python #!/usr/bin/env python
# expand_fluxes.py # expand_fluxes.py
import sys import sys
sys.path.append('../../')
import os import os
import getopt import getopt
from datetime import datetime, timedelta import shutil
from da.tools.general import create_dirs import logging
import netCDF4
import numpy as np import numpy as np
from string import join
from datetime import datetime, timedelta
from pylab import date2num, num2date from pylab import date2num, num2date
sys.path.append('../../')
from da.tools.general import create_dirs
import da.tools.io4 as io
""" """
Author: Wouter Peters (Wouter.Peters@wur.nl) Author: Wouter Peters (Wouter.Peters@wur.nl)
...@@ -33,15 +39,8 @@ def write_mixing_ratios(DaCycle): ...@@ -33,15 +39,8 @@ def write_mixing_ratios(DaCycle):
(4) Open the copied file, find the index of each observation, fill in the simulated data (4) Open the copied file, find the index of each observation, fill in the simulated data
""" """
import da.tools.io4 as io
from string import join
import shutil
import logging
import netCDF4
dirname = create_dirs(os.path.join(DaCycle['dir.analysis'], 'data_molefractions'))
dirname = 'data_molefractions'
dirname = create_dirs(os.path.join(DaCycle['dir.analysis'], dirname))
# #
# Some help variables # Some help variables
# #
......
...@@ -23,14 +23,18 @@ by oceans, sea, or open water. The aggregation will thus work best on arrays tha ...@@ -23,14 +23,18 @@ by oceans, sea, or open water. The aggregation will thus work best on arrays tha
""" """
import sys import sys
sys.path.append('../../') import cPickle
import os
from numpy import sum, array
try: try:
from dbfpy import dbf from dbfpy import dbf
except: except:
print "the python DBF lib might be needed, please install from:" print "the python DBF lib might be needed, please install from:"
print "http://dbfpy.sourceforge.net/" print "http://dbfpy.sourceforge.net/"
print " Trying to complete anyway..." print " Trying to complete anyway..."
from numpy import sum, array
sys.path.append('../../')
from da.analysis.tools_regions import globarea from da.analysis.tools_regions import globarea
EU25 = ["Austria", "Belgium", "Cyprus", "Czech Republic", "Denmark", "Estonia", "Finland", "France", "Germany", "Greece", "Hungary", "Ireland", "Italy", "Latvia", "Lithuania", "Luxembourg", "Malta", "Netherlands", "Poland", "Portugal", "Slovakia", "Slovenia", "Spain", "Sweden", "United Kingdom"] EU25 = ["Austria", "Belgium", "Cyprus", "Czech Republic", "Denmark", "Estonia", "Finland", "France", "Germany", "Greece", "Hungary", "Ireland", "Italy", "Latvia", "Lithuania", "Luxembourg", "Malta", "Netherlands", "Poland", "Portugal", "Slovakia", "Slovenia", "Spain", "Sweden", "United Kingdom"]
...@@ -129,8 +133,6 @@ def fix_eu(rec): ...@@ -129,8 +133,6 @@ def fix_eu(rec):
def get_countrydict(): def get_countrydict():
""" Create a dictionary with grid-to-country information from a dbf file""" """ Create a dictionary with grid-to-country information from a dbf file"""
import cPickle
import os
countrydict = countryinfo('Test') countrydict = countryinfo('Test')
......
#!/usr/bin/env python #!/usr/bin/env python
import commands
import os import numpy as np
import sys import cPickle
from da.analysis.tools_transcom import * from da.analysis.tools_transcom import *
# Aggregated olson ecosystem regions for CT Europe # Aggregated olson ecosystem regions for CT Europe
...@@ -31,21 +31,15 @@ def state_to_grid(values, regionmap, reverse=False, avg=False, mapname=None): ...@@ -31,21 +31,15 @@ def state_to_grid(values, regionmap, reverse=False, avg=False, mapname=None):
can subsequently be used in the transport model code to multiply/manipulate fluxes can subsequently be used in the transport model code to multiply/manipulate fluxes
""" """
import numpy as np
import cPickle
nregions = regionmap.max() nregions = regionmap.max()
try: try:
if not mapname: if not mapname:
raise Exception raise Exception
regionselect = cPickle.load(open('%s_regiondict.pickle' % mapname, 'rb')) regionselect = cPickle.load(open('%s_regiondict.pickle' % mapname, 'rb'))
except: except:
# dictionary for region <-> map conversions # dictionary for region <-> map conversions
regs = {} regs = {}
for r in np.arange(1, nregions + 1): for r in np.arange(1, nregions + 1):
sel = (regionmap.flat == r).nonzero() sel = (regionmap.flat == r).nonzero()
...@@ -53,14 +47,11 @@ def state_to_grid(values, regionmap, reverse=False, avg=False, mapname=None): ...@@ -53,14 +47,11 @@ def state_to_grid(values, regionmap, reverse=False, avg=False, mapname=None):
regs[r] = sel regs[r] = sel
regionselect = regs regionselect = regs
cPickle.dump(regionselect, open('%s_regiondict.pickle' % mapname, 'wb'), -1) cPickle.dump(regionselect, open('%s_regiondict.pickle' % mapname, 'wb'), -1)
print 'Pickling region map' print 'Pickling region map'
if reverse: if reverse:
""" project 1x1 degree map onto ecoregions """ """ project 1x1 degree map onto ecoregions """
result = np.zeros(nregions, float) result = np.zeros(nregions, float)
...@@ -72,7 +63,6 @@ def state_to_grid(values, regionmap, reverse=False, avg=False, mapname=None): ...@@ -72,7 +63,6 @@ def state_to_grid(values, regionmap, reverse=False, avg=False, mapname=None):
return result return result
else: else:
""" project ecoregion properties onto 1x1 degree map """ """ project ecoregion properties onto 1x1 degree map """
result = np.zeros((180, 360,), float) result = np.zeros((180, 360,), float)
...@@ -83,7 +73,6 @@ def state_to_grid(values, regionmap, reverse=False, avg=False, mapname=None): ...@@ -83,7 +73,6 @@ def state_to_grid(values, regionmap, reverse=False, avg=False, mapname=None):
def globarea(im=360, jm=180, silent=True): def globarea(im=360, jm=180, silent=True):
""" Function calculates the surface area according to TM5 definitions""" """ Function calculates the surface area according to TM5 definitions"""
import numpy as np
radius = 6.371e6 # the earth radius in meters radius = 6.371e6 # the earth radius in meters
deg2rad = np.pi / 180. deg2rad = np.pi / 180.
......
#! /usr/bin/env python #! /usr/bin/env python
import sys import sys
from datetime import datetime, timedelta
import calendar import calendar
from pylab import floor import copy
from datetime import datetime, timedelta
from matplotlib.dates import date2num, num2date from matplotlib.dates import date2num, num2date
from numpy import array, zeros, newaxis from numpy import array, zeros, newaxis, logical_and, arange
from pylab import floor, drange, num2date, date2num
def Fromdatetime(date): def Fromdatetime(date):
dt = date.timetuple() dt = date.timetuple()
...@@ -111,11 +113,6 @@ def nextmonth(dd): ...@@ -111,11 +113,6 @@ def nextmonth(dd):
def in_interval(start, stop, times_in): def in_interval(start, stop, times_in):
""" returns a list of fractions in time interval """ """ returns a list of fractions in time interval """
from numpy import logical_and, arange
from pylab import drange, num2date, date2num
import copy
times = copy.copy(times_in) times = copy.copy(times_in)
interval = times[1] - times[0] interval = times[1] - times[0]
......
...@@ -11,14 +11,21 @@ File created on 28 Jul 2010. ...@@ -11,14 +11,21 @@ File created on 28 Jul 2010.
import os import os
import sys import sys
import logging import logging
import datetime as dtm
from string import strip
from numpy import array, logical_and
sys.path.append(os.getcwd()) sys.path.append(os.getcwd())
sys.path.append('../../') sys.path.append('../../')
import da.tools.io4 as io
from da.baseclasses.obs import Observation
identifier = 'CarbonTracker CO2 mixing ratios' identifier = 'CarbonTracker CO2 mixing ratios'
version = '0.0' version = '0.0'
from da.baseclasses.obs import Observation
################### Begin Class ObsPackObservations ################### ################### Begin Class ObsPackObservations ###################
...@@ -50,10 +57,7 @@ class ObsPackObservations(Observation): ...@@ -50,10 +57,7 @@ class ObsPackObservations(Observation):
We will loop over all site files in the ObsPackage, and subset each to our needs We will loop over all site files in the ObsPackage, and subset each to our needs
""" """
import da.tools.io4 as io
import datetime as dtm
from string import strip
from numpy import array, logical_and
# Step 1: Read list of available site files in package # Step 1: Read list of available site files in package
...@@ -111,8 +115,6 @@ class ObsPackObservations(Observation): ...@@ -111,8 +115,6 @@ class ObsPackObservations(Observation):
def add_simulations(self, filename, silent=False): def add_simulations(self, filename, silent=False):
""" Adds model simulated values to the mixing ratio objects """ """ Adds model simulated values to the mixing ratio objects """
import da.tools.io4 as io
if not os.path.exists(filename): if not os.path.exists(filename):
msg = "Sample output filename for observations could not be found : %s" % filename msg = "Sample output filename for observations could not be found : %s" % filename
logging.error(msg) logging.error(msg)
...@@ -152,7 +154,6 @@ class ObsPackObservations(Observation): ...@@ -152,7 +154,6 @@ class ObsPackObservations(Observation):
Write the information needed by the observation operator to a file. Return the filename that was written for later use Write the information needed by the observation operator to a file. Return the filename that was written for later use
""" """
import da.tools.io4 as io
obsinputfile = os.path.join(dirinput, 'observations_%s.nc' % timestamp) obsinputfile = os.path.join(dirinput, 'observations_%s.nc' % timestamp)
...@@ -344,7 +345,6 @@ class ObsPackObservations(Observation): ...@@ -344,7 +345,6 @@ class ObsPackObservations(Observation):
Write selected information contained in the Observation object to a file. Write selected information contained in the Observation object to a file.
""" """
import da.tools.io4 as io
outfile = os.path.join(outdir, 'sampleinfo_%s.nc' % timestamp) outfile = os.path.join(outdir, 'sampleinfo_%s.nc' % timestamp)
......
...@@ -14,8 +14,10 @@ import sys ...@@ -14,8 +14,10 @@ import sys
sys.path.append(os.getcwd()) sys.path.append(os.getcwd())
import logging import logging
from da.baseclasses.statevector import StateVector
import numpy as np import numpy as np
from da.baseclasses.statevector import StateVector, EnsembleMember
import da.tools.io4 as io
identifier = 'CarbonTracker Statevector ' identifier = 'CarbonTracker Statevector '
version = '0.0' version = '0.0'
...@@ -30,8 +32,6 @@ class CtStateVector(StateVector): ...@@ -30,8 +32,6 @@ class CtStateVector(StateVector):
Note that lag=1 means an index of 0 in python, hence the notation lag-1 in the indexing below. Note that lag=1 means an index of 0 in python, hence the notation lag-1 in the indexing below.
The argument is thus referring to the lagged state vector as [1,2,3,4,5,..., nlag] The argument is thus referring to the lagged state vector as [1,2,3,4,5,..., nlag]
""" """
import da.tools.io4 as io
try: try:
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
except: except:
...@@ -109,9 +109,7 @@ class CtStateVector(StateVector): ...@@ -109,9 +109,7 @@ class CtStateVector(StateVector):
:meth:`~da.baseclasses.statevector.StateVector.write_to_file` :meth:`~da.baseclasses.statevector.StateVector.write_to_file`
""" """
import da.tools.io4 as io
from da.baseclasses.statevector import EnsembleMember
f = io.CT_Read(filename, 'read') f = io.CT_Read(filename, 'read')
......
...@@ -9,18 +9,15 @@ File created on 06 Sep 2010. ...@@ -9,18 +9,15 @@ File created on 06 Sep 2010.
""" """
import sys
import os
import logging import logging
import subprocess
from da.baseclasses.platform import PlatForm, std_joboptions from da.baseclasses.platform import PlatForm, std_joboptions
class CapeGrimPlatForm(PlatForm): class CapeGrimPlatForm(PlatForm):
def __init__(self): def __init__(self):
self.Identifier = 'WU capegrim' # the identifier gives the platform name self.Identifier = 'WU capegrim' # the identifier gives the platform name
self.Version = '1.0' # the platform version used self.Version = '1.0' # the platform version used
def give_blocking_flag(self): def give_blocking_flag(self):
return "" return ""
...@@ -28,7 +25,7 @@ class CapeGrimPlatForm(PlatForm): ...@@ -28,7 +25,7 @@ class CapeGrimPlatForm(PlatForm):
def give_queue_type(self): def give_queue_type(self):
return "foreground" return "foreground"
def get_job_template(self,joboptions={},block=False): def get_job_template(self, joboptions={}, block=False):
""" """
Returns the job template for a given computing system, and fill it with options from the dictionary provided as argument. Returns the job template for a given computing system, and fill it with options from the dictionary provided as argument.
The job template should return the preamble of a job that can be submitted to a queue on your platform, The job template should return the preamble of a job that can be submitted to a queue on your platform,
...@@ -46,38 +43,38 @@ class CapeGrimPlatForm(PlatForm): ...@@ -46,38 +43,38 @@ class CapeGrimPlatForm(PlatForm):
job until the submitted job in this template has been completed fully. job until the submitted job in this template has been completed fully.
""" """
template = """## \n"""+ \ template = """## \n""" + \
"""## This is a set of dummy names, to be replaced by values from the dictionary \n"""+ \ """## This is a set of dummy names, to be replaced by values from the dictionary \n""" + \
"""## Please make your own platform specific template with your own keys and place it in a subfolder of the da package.\n """+ \ """## Please make your own platform specific template with your own keys and place it in a subfolder of the da package.\n """ + \
"""## \n"""+ \ """## \n""" + \
""" \n"""+ \ """ \n""" + \
"""#$ jobname \n"""+ \ """#$ jobname \n""" + \
"""#$ jobaccount \n"""+ \ """#$ jobaccount \n""" + \
"""#$ jobnodes \n"""+ \ """#$ jobnodes \n""" + \
"""#$ jobtime \n"""+ \ """#$ jobtime \n""" + \
"""#$ jobshell \n"""+ \ """#$ jobshell \n""" + \
"""\n"""+ \ """\n""" + \
"""source /usr/local/Modules/3.2.8/init/sh\n"""+ \ """source /usr/local/Modules/3.2.8/init/sh\n""" + \
"""module load python\n"""+ \ """module load python\n""" + \
"""\n""" """\n"""
if 'depends' in joboptions: if 'depends' in joboptions:
template += """#$ -hold_jid depends \n""" template += """#$ -hold_jid depends \n"""
# First replace from passed dictionary # First replace from passed dictionary
for k,v in joboptions.iteritems(): for k, v in joboptions.iteritems():
while k in template: while k in template:
template = template.replace(k,v) template = template.replace(k, v)
# Fill remaining values with std_options # Fill remaining values with std_options
for k,v in std_joboptions.iteritems(): for k, v in std_joboptions.iteritems():
while k in template: while k in template:
template = template.replace(k,v) template = template.replace(k, v)
return template return template
msg1 = 'Platform initialized: %s'%self.Identifier ; logging.info(msg1) msg1 = 'Platform initialized: %s' % self.Identifier ; logging.info(msg1)
#msg2 = '%s version: %s'%(self.Identifier,self.Version) ; logging.info(msg2) #msg2 = '%s version: %s'%(self.Identifier,self.Version) ; logging.info(msg2)
......
...@@ -62,20 +62,20 @@ class HuygensPlatForm(PlatForm): ...@@ -62,20 +62,20 @@ class HuygensPlatForm(PlatForm):
job until the submitted job in this template has been completed fully. job until the submitted job in this template has been completed fully.
""" """
# template = """## \n"""+ \ #template = """## \n"""+ \
# """## This is a set of dummy names, to be replaced by values from the dictionary \n"""+ \ # """## This is a set of dummy names, to be replaced by values from the dictionary \n"""+ \
# """## Please make your own platform specific template with your own keys and place it in a subfolder of the da package.\n """+ \ # """## Please make your own platform specific template with your own keys and place it in a subfolder of the da package.\n """+ \
# """## \n"""+ \ # """## \n"""+ \
# """ \n"""+ \ # """ \n"""+ \
# """#$ jobname \n"""+ \ # """#$ jobname \n"""+ \
# """#$ jobaccount \n"""+ \ # """#$ jobaccount \n"""+ \