Skip to content
GitLab
Commit 3c8e0ac4 authored by karolina's avatar karolina
Browse files

ordered imports (mainly moved to the top of the module)

parent 0fd31898
Expand all Hide whitespace changes
Inline Side-by-side
gridded/da/analysis/expand_fluxes.py
View file @ 3c8e0ac4
......@@ -3,13 +3,23 @@
import sys
sys.path.append('../../')
import os
import getopt
from datetime import datetime, timedelta
from da.tools.general import create_dirs
from da.analysis.tools_country import countryinfo # needed here
import logging
import numpy as np
from pylab import date2num, num2date
import da.tools.io4 as io
from da.analysis.tools_regions import globarea, state_to_grid
from da.tools.general import create_dirs
from da.analysis.tools_country import countryinfo # needed here
from da.analysis.tools_transcom import transcommask, ExtendedTCRegions
import da.analysis.tools_transcom as tc
import da.analysis.tools_country as ct
import da.analysis.tools_time as timetools
"""
Author: Wouter Peters (Wouter.Peters@noaa.gov)
......@@ -38,10 +48,6 @@ def save_weekly_avg_1x1_data(DaCycle, StateVector):
:param StateVector: a :class:`~da.baseclasses.statevector.StateVector`
:rtype: None
"""
import da.tools.io4 as io
from da.analysis.tools_regions import globarea
#
dirname = create_dirs(os.path.join(DaCycle['dir.analysis'], 'data_flux1x1_weekly'))
#
......@@ -217,10 +223,6 @@ def save_weekly_avg_state_data(DaCycle, StateVector):
:param StateVector: a :class:`~da.baseclasses.statevector.StateVector`
:rtype: None
"""
import da.tools.io4 as io
import logging
from da.analysis.tools_regions import globarea
dirname = create_dirs(os.path.join(DaCycle['dir.analysis'], 'data_state_weekly'))
#
......@@ -452,9 +454,7 @@ def save_weekly_avg_tc_data(DaCycle, StateVector):
these with the parameters in the StateVector. This creates posterior fluxes, and the posterior covariance for the complete
StateVector in units of mol/box/s which we then turn into TC fluxes and covariances.
"""
from da.analysis.tools_regions import globarea
from da.analysis.tools_transcom import transcommask
import da.tools.io4 as io
#
dirname = create_dirs(os.path.join(DaCycle['dir.analysis'], 'data_tc_weekly'))
#
......@@ -584,9 +584,7 @@ def save_weekly_avg_ext_tc_data(DaCycle):
*** Example ***
./expand_savestate project=enkf_release sd=20000101 ed=20010101 """
from da.analysis.tools_transcom import ExtendedTCRegions
import da.tools.io4 as io
import logging
#
dirname = create_dirs(os.path.join(DaCycle['dir.analysis'], 'data_tc_weekly'))
#
......@@ -704,11 +702,7 @@ def save_weekly_avg_agg_data(DaCycle, region_aggregate='olson'):
these with the parameters in the StateVector. This creates posterior fluxes, and the posterior covariance for the complete
StateVector in units of mol/box/s which we then turn into TC fluxes and covariances.
"""
from da.analysis.tools_regions import globarea, state_to_grid
import da.analysis.tools_transcom as tc
import da.analysis.tools_country as ct
import da.tools.io4 as io
import logging
#
dirname = create_dirs(os.path.join(DaCycle['dir.analysis'], 'data_%s_weekly' % region_aggregate))
#
......@@ -918,9 +912,6 @@ def save_time_avg_data(DaCycle, infile, avg='monthly'):
*** Example ***
./expand_savestate project=enkf_release sd=20000101 ed=20010101 """
import da.analysis.tools_time as timetools
import da.tools.io4 as io
if 'weekly' in infile:
intime = 'weekly'
if 'monthly' in infile:
......
gridded/da/analysis/expand_mixingratios.py
View file @ 3c8e0ac4
#!/usr/bin/env python
# expand_fluxes.py
import sys
sys.path.append('../../')
import os
import getopt
from datetime import datetime, timedelta
from da.tools.general import create_dirs
import shutil
import logging
import netCDF4
import numpy as np
from string import join
from datetime import datetime, timedelta
from pylab import date2num, num2date
sys.path.append('../../')
from da.tools.general import create_dirs
import da.tools.io4 as io
"""
Author: Wouter Peters (Wouter.Peters@wur.nl)
......@@ -33,15 +39,8 @@ def write_mixing_ratios(DaCycle):
(4) Open the copied file, find the index of each observation, fill in the simulated data
"""
import da.tools.io4 as io
from string import join
import shutil
import logging
import netCDF4
dirname = 'data_molefractions'
dirname = create_dirs(os.path.join(DaCycle['dir.analysis'], dirname))
dirname = create_dirs(os.path.join(DaCycle['dir.analysis'], 'data_molefractions'))
#
# Some help variables
#
......
gridded/da/analysis/tools_country.py
View file @ 3c8e0ac4
......@@ -23,14 +23,18 @@ by oceans, sea, or open water. The aggregation will thus work best on arrays tha
"""
import sys
sys.path.append('../../')
import cPickle
import os
from numpy import sum, array
try:
from dbfpy import dbf
except:
print "the python DBF lib might be needed, please install from:"
print "http://dbfpy.sourceforge.net/"
print " Trying to complete anyway..."
from numpy import sum, array
sys.path.append('../../')
from da.analysis.tools_regions import globarea
EU25 = ["Austria", "Belgium", "Cyprus", "Czech Republic", "Denmark", "Estonia", "Finland", "France", "Germany", "Greece", "Hungary", "Ireland", "Italy", "Latvia", "Lithuania", "Luxembourg", "Malta", "Netherlands", "Poland", "Portugal", "Slovakia", "Slovenia", "Spain", "Sweden", "United Kingdom"]
......@@ -129,8 +133,6 @@ def fix_eu(rec):
def get_countrydict():
""" Create a dictionary with grid-to-country information from a dbf file"""
import cPickle
import os
countrydict = countryinfo('Test')
......
gridded/da/analysis/tools_regions.py
View file @ 3c8e0ac4
#!/usr/bin/env python
import commands
import os
import sys
import numpy as np
import cPickle
from da.analysis.tools_transcom import *
# Aggregated olson ecosystem regions for CT Europe
......@@ -31,21 +31,15 @@ def state_to_grid(values, regionmap, reverse=False, avg=False, mapname=None):
can subsequently be used in the transport model code to multiply/manipulate fluxes
"""
import numpy as np
import cPickle
nregions = regionmap.max()
try:
if not mapname:
raise Exception
regionselect = cPickle.load(open('%s_regiondict.pickle' % mapname, 'rb'))
except:
# dictionary for region <-> map conversions
regs = {}
for r in np.arange(1, nregions + 1):
sel = (regionmap.flat == r).nonzero()
......@@ -53,14 +47,11 @@ def state_to_grid(values, regionmap, reverse=False, avg=False, mapname=None):
regs[r] = sel
regionselect = regs
cPickle.dump(regionselect, open('%s_regiondict.pickle' % mapname, 'wb'), -1)
print 'Pickling region map'
if reverse:
""" project 1x1 degree map onto ecoregions """
result = np.zeros(nregions, float)
......@@ -72,7 +63,6 @@ def state_to_grid(values, regionmap, reverse=False, avg=False, mapname=None):
return result
else:
""" project ecoregion properties onto 1x1 degree map """
result = np.zeros((180, 360,), float)
......@@ -83,7 +73,6 @@ def state_to_grid(values, regionmap, reverse=False, avg=False, mapname=None):
def globarea(im=360, jm=180, silent=True):
""" Function calculates the surface area according to TM5 definitions"""
import numpy as np
radius = 6.371e6 # the earth radius in meters
deg2rad = np.pi / 180.
......
gridded/da/analysis/tools_time.py
View file @ 3c8e0ac4
#! /usr/bin/env python
import sys
from datetime import datetime, timedelta
import calendar
from pylab import floor
import copy
from datetime import datetime, timedelta
from matplotlib.dates import date2num, num2date
from numpy import array, zeros, newaxis
from numpy import array, zeros, newaxis, logical_and, arange
from pylab import floor, drange, num2date, date2num
def Fromdatetime(date):
dt = date.timetuple()
......@@ -111,11 +113,6 @@ def nextmonth(dd):
def in_interval(start, stop, times_in):
""" returns a list of fractions in time interval """
from numpy import logical_and, arange
from pylab import drange, num2date, date2num
import copy
times = copy.copy(times_in)
interval = times[1] - times[0]
......
gridded/da/ct/obspack.py
View file @ 3c8e0ac4
......@@ -11,14 +11,21 @@ File created on 28 Jul 2010.
import os
import sys
import logging
import datetime as dtm
from string import strip
from numpy import array, logical_and
sys.path.append(os.getcwd())
sys.path.append('../../')
import da.tools.io4 as io
from da.baseclasses.obs import Observation
identifier = 'CarbonTracker CO2 mixing ratios'
version = '0.0'
from da.baseclasses.obs import Observation
################### Begin Class ObsPackObservations ###################
......@@ -50,10 +57,7 @@ class ObsPackObservations(Observation):
We will loop over all site files in the ObsPackage, and subset each to our needs
"""
import da.tools.io4 as io
import datetime as dtm
from string import strip
from numpy import array, logical_and
# Step 1: Read list of available site files in package
......@@ -111,8 +115,6 @@ class ObsPackObservations(Observation):
def add_simulations(self, filename, silent=False):
""" Adds model simulated values to the mixing ratio objects """
import da.tools.io4 as io
if not os.path.exists(filename):
msg = "Sample output filename for observations could not be found : %s" % filename
logging.error(msg)
......@@ -152,7 +154,6 @@ class ObsPackObservations(Observation):
Write the information needed by the observation operator to a file. Return the filename that was written for later use
"""
import da.tools.io4 as io
obsinputfile = os.path.join(dirinput, 'observations_%s.nc' % timestamp)
......@@ -344,7 +345,6 @@ class ObsPackObservations(Observation):
Write selected information contained in the Observation object to a file.
"""
import da.tools.io4 as io
outfile = os.path.join(outdir, 'sampleinfo_%s.nc' % timestamp)
......
gridded/da/ct/statevector.py
View file @ 3c8e0ac4
......@@ -14,8 +14,10 @@ import sys
sys.path.append(os.getcwd())
import logging
from da.baseclasses.statevector import StateVector
import numpy as np
from da.baseclasses.statevector import StateVector, EnsembleMember
import da.tools.io4 as io
identifier = 'CarbonTracker Statevector '
version = '0.0'
......@@ -30,8 +32,6 @@ class CtStateVector(StateVector):
Note that lag=1 means an index of 0 in python, hence the notation lag-1 in the indexing below.
The argument is thus referring to the lagged state vector as [1,2,3,4,5,..., nlag]
"""
import da.tools.io4 as io
try:
import matplotlib.pyplot as plt
except:
......@@ -109,9 +109,7 @@ class CtStateVector(StateVector):
:meth:`~da.baseclasses.statevector.StateVector.write_to_file`
"""
import da.tools.io4 as io
from da.baseclasses.statevector import EnsembleMember
f = io.CT_Read(filename, 'read')
......
gridded/da/platform/capegrim.py
View file @ 3c8e0ac4
......@@ -9,18 +9,15 @@ File created on 06 Sep 2010.
"""
import sys
import os
import logging
import subprocess
from da.baseclasses.platform import PlatForm, std_joboptions
class CapeGrimPlatForm(PlatForm):
def __init__(self):
self.Identifier = 'WU capegrim' # the identifier gives the platform name
self.Version = '1.0' # the platform version used
self.Identifier = 'WU capegrim' # the identifier gives the platform name
self.Version = '1.0' # the platform version used
def give_blocking_flag(self):
return ""
......@@ -28,7 +25,7 @@ class CapeGrimPlatForm(PlatForm):
def give_queue_type(self):
return "foreground"
def get_job_template(self,joboptions={},block=False):
def get_job_template(self, joboptions={}, block=False):
"""
Returns the job template for a given computing system, and fill it with options from the dictionary provided as argument.
The job template should return the preamble of a job that can be submitted to a queue on your platform,
......@@ -46,38 +43,38 @@ class CapeGrimPlatForm(PlatForm):
job until the submitted job in this template has been completed fully.
"""
template = """## \n"""+ \
"""## This is a set of dummy names, to be replaced by values from the dictionary \n"""+ \
"""## Please make your own platform specific template with your own keys and place it in a subfolder of the da package.\n """+ \
"""## \n"""+ \
""" \n"""+ \
"""#$ jobname \n"""+ \
"""#$ jobaccount \n"""+ \
"""#$ jobnodes \n"""+ \
"""#$ jobtime \n"""+ \
"""#$ jobshell \n"""+ \
"""\n"""+ \
"""source /usr/local/Modules/3.2.8/init/sh\n"""+ \
"""module load python\n"""+ \
template = """## \n""" + \
"""## This is a set of dummy names, to be replaced by values from the dictionary \n""" + \
"""## Please make your own platform specific template with your own keys and place it in a subfolder of the da package.\n """ + \
"""## \n""" + \
""" \n""" + \
"""#$ jobname \n""" + \
"""#$ jobaccount \n""" + \
"""#$ jobnodes \n""" + \
"""#$ jobtime \n""" + \
"""#$ jobshell \n""" + \
"""\n""" + \
"""source /usr/local/Modules/3.2.8/init/sh\n""" + \
"""module load python\n""" + \
"""\n"""
if 'depends' in joboptions:
template += """#$ -hold_jid depends \n"""
# First replace from passed dictionary
for k,v in joboptions.iteritems():
for k, v in joboptions.iteritems():
while k in template:
template = template.replace(k,v)
template = template.replace(k, v)
# Fill remaining values with std_options
for k,v in std_joboptions.iteritems():
for k, v in std_joboptions.iteritems():
while k in template:
template = template.replace(k,v)
template = template.replace(k, v)
return template
msg1 = 'Platform initialized: %s'%self.Identifier ; logging.info(msg1)
msg1 = 'Platform initialized: %s' % self.Identifier ; logging.info(msg1)
#msg2 = '%s version: %s'%(self.Identifier,self.Version) ; logging.info(msg2)
......
gridded/da/platform/huygens.py
View file @ 3c8e0ac4
......@@ -62,20 +62,20 @@ class HuygensPlatForm(PlatForm):
job until the submitted job in this template has been completed fully.
"""
# template = """## \n"""+ \
# """## This is a set of dummy names, to be replaced by values from the dictionary \n"""+ \
# """## Please make your own platform specific template with your own keys and place it in a subfolder of the da package.\n """+ \
# """## \n"""+ \
# """ \n"""+ \
# """#$ jobname \n"""+ \
# """#$ jobaccount \n"""+ \
# """#$ jobnodes \n"""+ \
# """#$ jobtime \n"""+ \
# """#$ jobshell \n"""+ \
# """\n"""+ \
# """source /usr/bin/sh\n"""+ \
# """module load python\n"""+ \
# """\n"""
#template = """## \n"""+ \
# """## This is a set of dummy names, to be replaced by values from the dictionary \n"""+ \
# """## Please make your own platform specific template with your own keys and place it in a subfolder of the da package.\n """+ \
# """## \n"""+ \
# """ \n"""+ \
# """#$ jobname \n"""+ \
# """#$ jobaccount \n"""+ \
# """#$ jobnodes \n"""+ \
# """#$ jobtime \n"""+ \
# """#$ jobshell \n"""+ \
# """\n"""+ \
# """source /usr/bin/sh\n"""+ \
# """module load python\n"""+ \
# """\n"""
template = """#!/bin/bash \n""" + \
......@@ -150,19 +150,20 @@ class HuygensPlatForm(PlatForm):
""" This method submits a jobfile to the queue, and returns the queue ID """
# cmd = ["llsubmit","-s",jobfile]
# msg = "A new task will be started (%s)"%cmd ; logging.info(msg)
#cmd = ["llsubmit","-s",jobfile]
#msg = "A new task will be started (%s)"%cmd ; logging.info(msg)
if block:
cmd = ["llsubmit", "-s", jobfile]
msg = "A new task will be started (%s)" % cmd ; logging.info(msg)
output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] ; logging.info(output)
logging.info("A new task will be started (%s)" % cmd)
output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
logging.info(output)
print 'output', output
jobid = output.split()[3]
print 'jobid', jobid
else:
cmd = ["llsubmit", jobfile]
msg = "A new task will be started (%s)" % cmd ; logging.info(msg)
logging.info("A new task will be started (%s)" % cmd)
output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] ; logging.info(output)
jobid = output.split()[3]
......@@ -177,7 +178,7 @@ class HuygensPlatForm(PlatForm):
# jobid = output.split()[2]
# retcode = output.split()[-1]
#
#for huygens
# #for huygens
# print 'output', output
# test = output.split()[3]
# dummy, jobid =test.split('nl.')
......
gridded/da/platform/jet.py
View file @ 3c8e0ac4
......@@ -65,20 +65,18 @@ class JetPlatForm(PlatForm):
def submit_job(self, jobfile, joblog=None, block=False):
""" This method submits a jobfile to the queue, and returns the queue ID """
cmd = ["qsub", jobfile]
logging.info("A new task will be started (%s)" % cmd)
output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
logging.info(output)
jobid = output.split()[2]
#jobid = output.split()[2]
retcode = output.split()[-1]
return retcode
def kill_job(self, jobid):
""" This method kills a running job """
""" This method kills a running job """
output = subprocess.Popen(['qdel', jobid], stdout=subprocess.PIPE).communicate()[0]
logging.info(output)
return output
......
gridded/da/platform/maunaloa.py
View file @ 3c8e0ac4
......@@ -22,7 +22,6 @@ class MaunaloaPlatForm(PlatForm):
return ""
def give_queue_type(self):
return "foreground"
def get_job_template(self, joboptions={}, block=False):
......
gridded/da/tm5/observationoperator.py
View file @ 3c8e0ac4
......@@ -26,10 +26,13 @@ import logging
import shutil
import datetime
import subprocess
from string import join
sys.path.append(os.getcwd())
sys.path.append("../../")
from da.tools.general import create_dirs, to_datetime
from da.baseclasses.observationoperator import ObservationOperator
identifier = 'TM5'
version = 'release 3.0'
mpi_shell_filename = 'tm5_mpi_wrapper'
......@@ -38,7 +41,7 @@ mpi_shell_location = 'da/bin/'
################### Begin Class TM5 ###################
from da.baseclasses.observationoperator import ObservationOperator
class TM5ObservationOperator(ObservationOperator):
""" This class holds methods and variables that are needed to run the TM5 model. It is initiated with as only argument a TM5 rc-file
......@@ -87,7 +90,7 @@ class TM5ObservationOperator(ObservationOperator):
Execute all steps needed to prepare the ObsOperator for use inside CTDAS, only done at the very first cycle normally
"""
from da.tools.general import create_dirs
if self.DaCycle['time.restart'] == False :
logging.info('First time step, setting up and compiling the TM5 model before proceeding!')
......@@ -130,8 +133,7 @@ class TM5ObservationOperator(ObservationOperator):
"""
Compile TM5 model using setup_tm5 and the modified rc-file
"""
import subprocess
from string import join
DaPlatForm = self.DaCycle.DaPlatForm
if self.DaCycle.has_key('da.obsoperator.home'):
......@@ -228,7 +230,7 @@ class TM5ObservationOperator(ObservationOperator):
Validate the contents of the tm_settings dictionary and add extra values. The required items for the TM5 rc-file
are specified in the tm5_tools module, as dictionary variable "needed_rc_items".
"""
from da.tools.general import to_datetime
if self.RcFileType == 'pycasso':
self.projectkey = 'my.project.dir'
......
gridded/da/tools/initexit.py
View file @ 3c8e0ac4
......@@ -59,10 +59,13 @@ import os
import sys
import shutil
import copy
import getopt
import cPickle
import numpy as np
from string import join
import da.tools.rc as rc
from da.tools.general import create_dirs
from da.tools.general import create_dirs, to_datetime, advance_time
needed_da_items = [
'time.start',
......@@ -131,7 +134,6 @@ class CycleControl(dict):
Validate the contents of the rc-file given a dictionary of required keys.
Currently required keys are :attr:`~da.tools.initexit.needed_da_items`
"""
from da.tools.general import to_datetime
for k, v in self.iteritems():
if v in ['True', 'true', 't', 'T', 'y', 'yes']:
......@@ -161,7 +163,6 @@ class CycleControl(dict):
"""
Parse time related parameters into datetime objects for later use
"""
from da.tools.general import advance_time
startdate = self['time.start']
finaldate = self['time.finish']
......@@ -217,7 +218,6 @@ class CycleControl(dict):
"""
Advance sampling start and end time by one cycle interval
"""
from da.tools.general import advance_time
days = self['cyclelength'].days
......@@ -229,7 +229,6 @@ class CycleControl(dict):
"""
Advance cycle start and end time by one cycle interval
"""