Commit 8ffdbf8c authored by brunner's avatar brunner
Browse files

no cdo afternoon mean

parent 640e5e92
......@@ -29,17 +29,16 @@
! The time for which to start and end the data assimilation experiment in format YYYY-MM-DD HH:MM:SS
! the following 3 lines are for initial start
time.start : 2019-01-01 00:00:00
time.finish : 2019-01-07 23:00:00
time.end : 2019-01-07 23:00:00
abs.time.start : 2019-01-01 00:00:00
time.start : 2019-03-01 00:00:00
time.finish : 2019-03-07 23:00:00
time.end : 2019-03-07 23:00:00
abs.time.start : 2019-03-01 00:00:00
! Whether to restart the CTDAS system from a previous cycle, or to start the sequence fresh. Valid entries are T/F/True/False/TRUE/FALSE
time.restart : F
!da.restart.tstamp : 2013-01-08 00:00:00
da.restart.tstamp : 2013-01-01 00:00:00
! da.restart.tstamp : 2019-03-08 00:00:00
da.restart.tstamp : 2019-03-01 00:00:00
! The length of a cycle is given in days, such that the integer 7 denotes the typically used weekly cycle. Valid entries are integers > 1
......@@ -54,6 +53,7 @@ time.nlag : 2
run.name : real
dir.da_run : /scratch/snx3000/parsenov/${run.name}
dir.ct_save : /store/empa/em05/parsenov/ct_data/${run.name}/
restartmap.dir : ${dir.da_run}/input
! The resources used to complete the data assimilation experiment. This depends on your computing platform.
......@@ -86,6 +86,7 @@ da.system : CarbonTracker
! The specific settings for your system are read from a separate rc-file, which points to the data directories, observations, etc
da.system.rc : da/rc/carbontracker_cosmo.rc
locations : /store/empa/em05/parsenov/ct_data/locations.csv
! This flag should probably be moved to the da.system.rc file. It denotes which type of filtering to use in the optimizer
......
......@@ -85,5 +85,3 @@ save_weekly_avg_ext_tc_data(dacycle)
write_mole_fractions(dacycle)
sys.exit(0)
......@@ -313,6 +313,8 @@ class Optimizer(object):
def serial_minimum_least_squares(self):
""" Make minimum least squares solution by looping over obs"""
# Corrected for bias over all stations
bias = np.mean(self.obs) - np.mean(self.Hx)
for n in range(self.nobs):
# Screen for flagged observations (for instance site not found, or no sample written from model)
......@@ -322,13 +324,15 @@ class Optimizer(object):
continue
# Screen for outliers greather than 3x model-data mismatch, only apply if obs may be rejected
# Calculate residual for rejecting the observations (corrected for bias) - res_rej
res_rej = self.obs[n] - self.Hx[n] - bias
res = self.obs[n] - self.Hx[n]
if self.may_reject[n]:
threshold = self.rejection_threshold * np.sqrt(self.R[n])
if np.abs(res) > threshold:
logging.debug('Rejecting observation (%s,%i) because residual (%f) exceeds threshold (%f)' % (self.sitecode[n], self.obs_ids[n], res, threshold))
#if np.abs(res) > threshold + abs(bias):
if np.abs(res_rej) > threshold:
logging.debug('Rejecting observation (%s,%i) because residual (%f) exceeds threshold (%f)' % (self.sitecode[n], self.obs_ids[n], res, threshold + abs(bias)))
self.flags[n] = 2
continue
......
......@@ -64,17 +64,29 @@ class CO2StateVector(StateVector):
fullcov = np.zeros(shape=(90,90))
# partcov = np.array([ \
# (0.64, 0.36, 0.16, 0.16, 0.16, 0.16, 0.04, 0.04, 0.04, 0.01), \
# (0.36, 0.64, 0.16, 0.16, 0.16, 0.16, 0.04, 0.04, 0.04, 0.01), \
# (0.16, 0.16, 0.64, 0.36, 0.16, 0.16, 0.04, 0.04, 0.04, 0.01), \
# (0.16, 0.16, 0.36, 0.64, 0.16, 0.16, 0.04, 0.04, 0.04, 0.01), \
# (0.16, 0.16, 0.16, 0.16, 0.64, 0.36, 0.04, 0.04, 0.04, 0.01), \
# (0.16, 0.16, 0.16, 0.16, 0.36, 0.64, 0.04, 0.04, 0.04, 0.01), \
# (0.04, 0.04, 0.04, 0.04, 0.04, 0.04, 0.64, 0.16, 0.16, 0.16), \
#(0.04, 0.04, 0.04, 0.04, 0.04, 0.04, 0.16, 0.64, 0.16, 0.16), \
# (0.04, 0.04, 0.04, 0.04, 0.04, 0.04, 0.16, 0.16, 0.64, 0.16), \
# (0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.16, 0.16, 0.16, 0.64) ])
partcov = np.array([ \
(0.64, 0.36, 0.16, 0.16, 0.16, 0.16, 0.04, 0.04, 0.04, 0.01), \
(0.36, 0.64, 0.16, 0.16, 0.16, 0.16, 0.04, 0.04, 0.04, 0.01), \
(0.16, 0.16, 0.64, 0.36, 0.16, 0.16, 0.04, 0.04, 0.04, 0.01), \
(0.16, 0.16, 0.36, 0.64, 0.16, 0.16, 0.04, 0.04, 0.04, 0.01), \
(0.16, 0.16, 0.16, 0.16, 0.64, 0.36, 0.04, 0.04, 0.04, 0.01), \
(0.16, 0.16, 0.16, 0.16, 0.36, 0.64, 0.04, 0.04, 0.04, 0.01), \
(0.04, 0.04, 0.04, 0.04, 0.04, 0.04, 0.64, 0.16, 0.16, 0.16), \
(0.04, 0.04, 0.04, 0.04, 0.04, 0.04, 0.16, 0.64, 0.16, 0.16), \
(0.04, 0.04, 0.04, 0.04, 0.04, 0.04, 0.16, 0.16, 0.64, 0.16), \
(0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.16, 0.16, 0.16, 0.64) ])
(0.1089, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900), \
(0.0900, 0.1089, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900), \
(0.0900, 0.0900, 0.1089, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900), \
(0.0900, 0.0900, 0.0900, 0.1089, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900), \
(0.0900, 0.0900, 0.0900, 0.0900, 0.1089, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900), \
(0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.1089, 0.0900, 0.0900, 0.0900, 0.0900), \
(0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.1089, 0.0900, 0.0900, 0.0900), \
(0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.1089, 0.0900, 0.0900), \
(0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.1089, 0.0900), \
(0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.0900, 0.1089) ])
# L = 300 km
......
......@@ -5,23 +5,22 @@ import logging
import os
import sys
import subprocess
import csv
import da.cosmo.io4 as io
import numpy as np
from netCDF4 import Dataset
from scipy import interpolate
from datetime import datetime, timedelta
from dateutil import rrule
from cdo import *
from . import site_height
from da.cosmo.icbc4ctdas import ct
from itertools import repeat
from multiprocessing import Pool
from da.tools.general import to_datetime
import amrs.misc.transform as transform
identifier = 'ObservationOperator'
version = '10'
#cdo = Cdo()
cdo = Cdo(logging=True, logFile='cdo_commands.log')
################### Begin Class ObservationOperator ###################
class ObservationOperator(object):
......@@ -57,15 +56,19 @@ class ObservationOperator(object):
self.simulated_file = os.path.join(self.outputdir, 'samples_simulated.%s.nc' % self.dacycle['time.sample.stamp'])
self.forecast_nmembers = int(self.dacycle['da.optimizer.nmembers'])
def run(self,lag,dacycle,statevector,advance=False):
def run(self, lag, dacycle, statevector, advance=False):
members = statevector.ensemble_members[lag]
self.forecast_nmembers = int(self.dacycle['da.optimizer.nmembers'])
self.nparams = int(self.dacycle['nparameters'])
absolute_start_time = str((to_datetime(dacycle['abs.time.start'])).strftime('%Y%m%d%H'))
absolute_start_time_ch = str((to_datetime(dacycle['abs.time.start'])).strftime('%Y-%m-%d'))
self.days = int(dacycle['time.cycle'])
abs_start_time = str((to_datetime(dacycle['abs.time.start'])).strftime('%Y%m%d%H'))
abs_start_time_ch = str((to_datetime(dacycle['abs.time.start'])).strftime('%Y-%m-%d'))
starth = abs((to_datetime(dacycle['abs.time.start'])-dacycle['time.start']).days)*24
endh = abs((to_datetime(dacycle['abs.time.start'])-dacycle['time.finish']).days)*24
start = dacycle['time.start']
end = dacycle['time.finish']
f = io.CT_CDF(self.simulated_file, method='create')
logging.debug('Creating new simulated observation file in ObservationOperator (%s)' % self.simulated_file)
......@@ -116,36 +119,41 @@ class ObservationOperator(object):
for m in range(0,self.forecast_nmembers):
co2[m,:] = members[m].param_values
# co2[co2<0] = 0.
l[:] = co2
ofile.close()
os.system('cp '+self.lambda_file+' '+dacycle['da.vprm']+'/lambdas.nc')
os.chdir(dacycle['da.obsoperator.home'])
if os.path.exists(dacycle['dir.da_run']+'/'+absolute_start_time+"_"+str(starth+lag*168)+"_"+str(endh+lag*168)+"/cosmo/output/"):
if os.path.exists(dacycle['dir.da_run']+"/non_opt_"+absolute_start_time+"_"+str(starth+lag*168)+"_"+str(endh+lag*168)+"/cosmo/output/"):
os.rename(dacycle['dir.da_run']+"/"+absolute_start_time+"_"+str(starth+lag*168)+"_"+str(endh+lag*168), dacycle['dir.da_run']+"/old_non_opt_"+dacycle['time.start'].strftime('%Y%m%d%H')+"_"+str(starth+lag*168)+"_"+str(endh+lag*168))
else:
os.rename(dacycle['dir.da_run']+"/"+absolute_start_time+"_"+str(starth+lag*168)+"_"+str(endh+lag*168), dacycle['dir.da_run']+"/non_opt_"+dacycle['time.start'].strftime('%Y%m%d%H')+"_"+str(starth+lag*168)+"_"+str(endh+lag*168))
logging.info('Starting COSMO')
os.system('python run_chain.py '+self.dacycle['run.name']+' '+abs_start_time_ch+' '+str(starth+lag*168)+' '+str(endh+lag*168)+' -j meteo icbc int2lm post_int2lm oae octe online_vprm cosmo')
os.system('python run_chain.py '+self.dacycle['run.name']+' '+absolute_start_time_ch+' '+str(starth+lag*168)+' '+str(endh+lag*168)+' -j meteo icbc int2lm post_int2lm oae octe online_vprm cosmo')
logging.info('COSMO done!')
os.chdir(dacycle['dir.da_run'])
# Here the extraction of COSMO output starts
dicts = self.read_csv(dacycle)
rlat, rlon, dicts, path_in = self.get_hhl_data(dacycle, lag, 'lffd'+abs_start_time+'c.nc', dicts, starth, endh)
logging.info('Starting parallel extraction \m/')
args = [
(dacycle, starth+168*lag, endh+168*lag-1, n)
for n in range(1,self.forecast_nmembers+1)
(dacycle, dacycle['time.sample.start']+timedelta(hours = 24*n), dicts, rlat, rlon, path_in)
for n in range(self.days)
]
with Pool(self.forecast_nmembers) as pool:
pool.starmap(self.extract_model_data, args)
with Pool(self.days) as pool:
pool.starmap(self.get_cosmo_data, args)
logging.info('Finished parallel extraction \m/')
self.cat_cosmo_data(advance, dacycle)
for i in range(0,self.forecast_nmembers):
idx = str(i+1).zfill(3)
cosmo_file = os.path.join('/store/empa/em05/parsenov/cosmo_data/model_'+idx+'_%s.nc' % dacycle['time.sample.stamp'])
cosmo_file = os.path.join(self.dacycle['dir.ct_save'], 'Hx_'+idx+'_%s.nc' % dacycle['time.sample.stamp'])
ifile = Dataset(cosmo_file, mode='r')
model_data[i,:] = (np.squeeze(ifile.variables['CO2'][:])*29./44.01)*1E6 # in ppm
model_data[i,:] = np.squeeze(ifile.variables['CO2'][:])
ifile.close()
for j,data in enumerate(zip(ids,obs,mdm)):
......@@ -159,195 +167,261 @@ class ObservationOperator(object):
self.prepare_run()
self.run(lag, dacycle, statevector, advance)
def extract_model_data(self, dacycle, hstart, hstop, ensnum):
self.dacycle = dacycle
time_stamp = dacycle['time.sample.stamp']
def read_csv(self, dacycle):
"""Reads csv file where information about stations is written"""
ensnum = int(dacycle['da.optimizer.nmembers'])
csvfile = dacycle['locations']
dicts = []
with open(csvfile) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
for row in csv_reader:
for e in range(ensnum):
e = str(e+1).zfill(3)
dicts = np.append(dicts, {'ensnum':e, 'name':row[1], 'lon':row[2], 'lat':row[3], \
'rlon':None, 'rlat':None, 'h1':None, 'h2':None, \
'hidx1':None, 'hidx2':None, \
'alt':float(row[4])+float(row[5]), 'time':[], 'co2':[], \
'co2_bg':[], 'co2_gpp':[], 'co2_ra':[], 'co2_a':[]})
return dicts
def get_hhl_data(self, dacycle, lag, ncc, dicts, starth, endh):
abs_start_time = str((to_datetime(dacycle['abs.time.start'])).strftime('%Y%m%d%H'))
path_in = os.path.join(dacycle['dir.da_run'], abs_start_time+'_'+str(starth+lag*168)+'_'+str(endh+lag*168), "cosmo/output/")
hhl = np.empty(shape=(60))
hhl60 = np.empty(shape=(60, 300, 450))
with Dataset(path_in+ncc) as nc1:
rotpole = nc1.variables['rotated_pole']
pollon = rotpole.getncattr('grid_north_pole_longitude')
pollat = rotpole.getncattr('grid_north_pole_latitude')
rlat = nc1.variables['rlat'][:]
rlon = nc1.variables['rlon'][:]
hhl_3d = np.squeeze(nc1.variables['HHL'][:])
for h in range(0,60):
hhl60[h, :, :]=(hhl_3d[h, :, :]+hhl_3d[h+1, :, :])/2.
for station in dicts:
myrlon, myrlat = transform.rotpole2wgs(float(station['lon']), \
float(station['lat']), \
pollon, pollat, inverse=True)
for h in range(0,60):
hhl[h] = interpolate.interpn((rlat,rlon), hhl60[h,:,:], [myrlat,myrlon], method='linear')
if float(station['alt']) < hhl[59]:
station['h1'] = hhl[59]
station['h2'] = hhl[59]
station['hidx1'] = 59
station['hidx2'] = 59
station['alt'] = hhl[59]
cosmo_out = dacycle['dir.da_run']+"/"+abs_start_time+"_"+str(hstart)+"_"+str(hstop+1)+"/cosmo/output/"
hhl_cosmo_out = dacycle['dir.da_run']+"/"+abs_start_time+"_0_168/cosmo/output/"
cosmo_save = "/store/empa/em05/parsenov/cosmo_data/"
hhl_fn = hhl_cosmo_out+'lffd'+abs_start_time+'c.nc'
ens = str(ensnum).zfill(3)
files2cat_albs = []
files2cat_bntg = []
files2cat_brm = []
files2cat_chri = []
files2cat_due1 = []
files2cat_esmo = []
files2cat_frob = []
files2cat_gimm = []
files2cat_hae = []
files2cat_laeg = []
files2cat_magn = []
files2cat_payn = []
files2cat_reck = []
files2cat_rig = []
files2cat_save = []
files2cat_semp = []
files2cat_sott = []
files2cat_ssal = []
files2cat_taen = []
files2cat_zhbr = []
files2cat_zsch = []
files2cat_zue = []
if ens == "001":
cdo.selname("HHL", input = hhl_fn, output = cosmo_out+"hhl.nc")
cdo.remapnn("lon=8.51_lat=47.31,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_albs.nc")
cdo.remapnn("lon=7.53_lat=46.98,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_bntg.nc")
cdo.remapnn("lon=8.18_lat=47.19,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_brm.nc")
cdo.remapnn("lon=7.69_lat=47.57,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_chri.nc")
cdo.remapnn("lon=8.61_lat=47.40,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_due1.nc")
cdo.remapnn("lon=8.57_lat=47.52,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_esmo.nc")
cdo.remapnn("lon=7.90_lat=47.38,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_frob.nc")
cdo.remapnn("lon=7.25_lat=47.05,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_gimm.nc")
cdo.remapnn("lon=7.82_lat=47.31,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_hae.nc")
cdo.remapnn("lon=8.40_lat=47.48,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_laeg.nc")
cdo.remapnn("lon=8.93_lat=46.16,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_magn.nc")
cdo.remapnn("lon=6.94_lat=46.81,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_payn.nc")
cdo.remapnn("lon=8.52_lat=47.43,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_reck.nc")
cdo.remapnn("lon=8.46_lat=47.07,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_rig.nc")
cdo.remapnn("lon=7.36_lat=46.24,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_save.nc")
cdo.remapnn("lon=8.21_lat=47.12,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_semp.nc")
cdo.remapnn("lon=6.74_lat=46.66,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_sott.nc")
cdo.remapnn("lon=8.95_lat=45.98,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_ssal.nc")
cdo.remapnn("lon=8.90_lat=47.48,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_taen.nc")
cdo.remapnn("lon=8.57_lat=47.38,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_zhbr.nc")
cdo.remapnn("lon=8.52_lat=47.37,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_zsch.nc")
cdo.remapnn("lon=8.53_lat=47.38,", input = cosmo_out+"hhl.nc", output = cosmo_out+"hhl_zue.nc")
for dt in rrule.rrule(rrule.HOURLY, dtstart=to_datetime(dacycle['abs.time.start'])+timedelta(hours=hstart), until=to_datetime(dacycle['abs.time.start'])+timedelta(hours=hstop)):
dt=dt.strftime('%Y%m%d%H')
if ens == "001":
logging.info('Extracting output for time %s' % (str(dt)))
co2_in_fn = cosmo_out+'lffd'+dt+'.nc'
co2_out_albs = cosmo_out+'CO2_albs_'+ens+'_'+dt+'.nc'
co2_out_bntg = cosmo_out+'CO2_bntg_'+ens+'_'+dt+'.nc'
co2_out_brm = cosmo_out+'CO2_brm_'+ens+'_'+dt+'.nc'
co2_out_chri = cosmo_out+'CO2_chri_'+ens+'_'+dt+'.nc'
co2_out_due1 = cosmo_out+'CO2_due1_'+ens+'_'+dt+'.nc'
co2_out_esmo = cosmo_out+'CO2_esmo_'+ens+'_'+dt+'.nc'
co2_out_frob = cosmo_out+'CO2_frob_'+ens+'_'+dt+'.nc'
co2_out_gimm = cosmo_out+'CO2_gimm_'+ens+'_'+dt+'.nc'
co2_out_hae = cosmo_out+'CO2_hae_'+ens+'_'+dt+'.nc'
co2_out_laeg = cosmo_out+'CO2_laeg_'+ens+'_'+dt+'.nc'
co2_out_magn = cosmo_out+'CO2_magn_'+ens+'_'+dt+'.nc'
co2_out_payn = cosmo_out+'CO2_payn_'+ens+'_'+dt+'.nc'
co2_out_reck = cosmo_out+'CO2_reck_'+ens+'_'+dt+'.nc'
co2_out_rig = cosmo_out+'CO2_rig_'+ens+'_'+dt+'.nc'
co2_out_save = cosmo_out+'CO2_save_'+ens+'_'+dt+'.nc'
co2_out_semp = cosmo_out+'CO2_semp_'+ens+'_'+dt+'.nc'
co2_out_sott = cosmo_out+'CO2_sott_'+ens+'_'+dt+'.nc'
co2_out_ssal = cosmo_out+'CO2_ssal_'+ens+'_'+dt+'.nc'
co2_out_taen = cosmo_out+'CO2_taen_'+ens+'_'+dt+'.nc'
co2_out_zhbr = cosmo_out+'CO2_zhbr_'+ens+'_'+dt+'.nc'
co2_out_zsch = cosmo_out+'CO2_zsch_'+ens+'_'+dt+'.nc'
co2_out_zue = cosmo_out+'CO2_zue_'+ens+'_'+dt+'.nc'
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=8.51_lat=47.31 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_albs)
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=7.53_lat=46.98 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_bntg)
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=8.18_lat=47.19 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_brm)
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=7.69_lat=47.57 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_chri)
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=8.61_lat=47.40 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_due1)
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=8.57_lat=47.52 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_esmo)
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=7.90_lat=47.38 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_frob)
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=7.25_lat=47.05 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_gimm)
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=7.82_lat=47.31 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_hae)
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=8.40_lat=47.48 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_laeg)
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=8.93_lat=46.16 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_magn)
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=6.94_lat=46.81 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_payn)
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=8.52_lat=47.43 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_reck)
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=8.46_lat=47.07 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_rig)
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=7.36_lat=46.24 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_save)
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=8.21_lat=47.12 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_semp)
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=6.74_lat=46.66 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_sott)
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=8.95_lat=45.98 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_ssal)
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=8.90_lat=47.48 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_taen)
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=8.57_lat=47.38 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_zhbr)
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=8.52_lat=47.37 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_zsch)
cdo.expr("'CO2=(CO2_BG"+ens+"-CO2_GPP"+ens+"+CO2_RA"+ens+"+CO2_A)/(1.-QV)'", input = "-remapnn,lon=8.53_lat=47.38 -selname,QV,CO2_BG"+ens+",CO2_GPP"+ens+",CO2_RA"+ens+",CO2_A "+co2_in_fn, output = co2_out_zue)
files2cat_albs.append(co2_out_albs)
files2cat_bntg.append(co2_out_bntg)
files2cat_brm.append(co2_out_brm)
files2cat_chri.append(co2_out_chri)
files2cat_due1.append(co2_out_due1)
files2cat_esmo.append(co2_out_esmo)
files2cat_frob.append(co2_out_frob)
files2cat_gimm.append(co2_out_gimm)
files2cat_hae.append(co2_out_hae)
files2cat_laeg.append(co2_out_laeg)
files2cat_magn.append(co2_out_magn)
files2cat_payn.append(co2_out_payn)
files2cat_reck.append(co2_out_reck)
files2cat_rig.append(co2_out_rig)
files2cat_save.append(co2_out_save)
files2cat_semp.append(co2_out_semp)
files2cat_sott.append(co2_out_sott)
files2cat_ssal.append(co2_out_ssal)
files2cat_taen.append(co2_out_taen)
files2cat_zhbr.append(co2_out_zhbr)
files2cat_zsch.append(co2_out_zsch)
files2cat_zue.append(co2_out_zue)
cdo.cat(input = files2cat_albs, output = cosmo_out+"CO2_albs_"+ens+"_"+time_stamp+".nc")
cdo.cat(input = files2cat_bntg, output = cosmo_out+"CO2_bntg_"+ens+"_"+time_stamp+".nc")
cdo.cat(input = files2cat_brm, output = cosmo_out+"CO2_brm_"+ens+"_"+time_stamp+".nc")
cdo.cat(input = files2cat_chri, output = cosmo_out+"CO2_chri_"+ens+"_"+time_stamp+".nc")
cdo.cat(input = files2cat_due1, output = cosmo_out+"CO2_due1_"+ens+"_"+time_stamp+".nc")
cdo.cat(input = files2cat_esmo, output = cosmo_out+"CO2_esmo_"+ens+"_"+time_stamp+".nc")
cdo.cat(input = files2cat_frob, output = cosmo_out+"CO2_frob_"+ens+"_"+time_stamp+".nc")
cdo.cat(input = files2cat_gimm, output = cosmo_out+"CO2_gimm_"+ens+"_"+time_stamp+".nc")
cdo.cat(input = files2cat_hae, output = cosmo_out+"CO2_hae_"+ens+"_"+time_stamp+".nc")
cdo.cat(input = files2cat_laeg, output = cosmo_out+"CO2_laeg_"+ens+"_"+time_stamp+".nc")
cdo.cat(input = files2cat_magn, output = cosmo_out+"CO2_magn_"+ens+"_"+time_stamp+".nc")
cdo.cat(input = files2cat_payn, output = cosmo_out+"CO2_payn_"+ens+"_"+time_stamp+".nc")
cdo.cat(input = files2cat_reck, output = cosmo_out+"CO2_reck_"+ens+"_"+time_stamp+".nc")
cdo.cat(input = files2cat_rig, output = cosmo_out+"CO2_rig_"+ens+"_"+time_stamp+".nc")
cdo.cat(input = files2cat_save, output = cosmo_out+"CO2_save_"+ens+"_"+time_stamp+".nc")
cdo.cat(input = files2cat_semp, output = cosmo_out+"CO2_semp_"+ens+"_"+time_stamp+".nc")
cdo.cat(input = files2cat_sott, output = cosmo_out+"CO2_sott_"+ens+"_"+time_stamp+".nc")
cdo.cat(input = files2cat_ssal, output = cosmo_out+"CO2_ssal_"+ens+"_"+time_stamp+".nc")
cdo.cat(input = files2cat_taen, output = cosmo_out+"CO2_taen_"+ens+"_"+time_stamp+".nc")
cdo.cat(input = files2cat_zhbr, output = cosmo_out+"CO2_zhbr_"+ens+"_"+time_stamp+".nc")
cdo.cat(input = files2cat_zsch, output = cosmo_out+"CO2_zsch_"+ens+"_"+time_stamp+".nc")
cdo.cat(input = files2cat_zue, output = cosmo_out+"CO2_zue_"+ens+"_"+time_stamp+".nc")
sites = ("albs", "bntg", "brm", "chri", "due1", "esmo", "frob", "gimm", "hae", "laeg", "magn", "payn", "reck", "rig", "save", "semp", "sott", "ssal", "taen", "zhbr", "zsch", "zue")
for s,ss in enumerate(sites):
site_height.main(cosmo_out, str(ens), ss, time_stamp)
cdo.intlevel("857.25", input = cosmo_out+"CO2_60lev_"+ens+"_albs_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_albs_"+time_stamp+".nc")
cdo.intlevel("995.35", input = cosmo_out+"CO2_60lev_"+ens+"_bntg_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_bntg_"+time_stamp+".nc")
cdo.intlevel("1009.7", input = cosmo_out+"CO2_60lev_"+ens+"_brm_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_brm_"+time_stamp+".nc")
cdo.intlevel("592.8", input = cosmo_out+"CO2_60lev_"+ens+"_chri_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_chri_"+time_stamp+".nc")
cdo.intlevel("432.4", input = cosmo_out+"CO2_60lev_"+ens+"_due1_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_due1_"+time_stamp+".nc")
cdo.intlevel("560.9", input = cosmo_out+"CO2_60lev_"+ens+"_esmo_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_esmo_"+time_stamp+".nc")
cdo.intlevel("897.3", input = cosmo_out+"CO2_60lev_"+ens+"_frob_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_frob_"+time_stamp+".nc")
cdo.intlevel("478.5", input = cosmo_out+"CO2_60lev_"+ens+"_gimm_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_gimm_"+time_stamp+".nc")
cdo.intlevel("430.2", input = cosmo_out+"CO2_60lev_"+ens+"_hae_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_hae_"+time_stamp+".nc")
cdo.intlevel("855", input = cosmo_out+"CO2_60lev_"+ens+"_laeg_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_laeg_"+time_stamp+".nc")
cdo.intlevel("208", input = cosmo_out+"CO2_60lev_"+ens+"_magn_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_magn_"+time_stamp+".nc")
cdo.intlevel("488.7", input = cosmo_out+"CO2_60lev_"+ens+"_payn_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_payn_"+time_stamp+".nc")
cdo.intlevel("443.1", input = cosmo_out+"CO2_60lev_"+ens+"_reck_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_reck_"+time_stamp+".nc")
cdo.intlevel("1030.5", input = cosmo_out+"CO2_60lev_"+ens+"_rig_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_rig_"+time_stamp+".nc")
cdo.intlevel("789.4", input = cosmo_out+"CO2_60lev_"+ens+"_save_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_save_"+time_stamp+".nc")
cdo.intlevel("583.3", input = cosmo_out+"CO2_60lev_"+ens+"_semp_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_semp_"+time_stamp+".nc")
cdo.intlevel("774.9", input = cosmo_out+"CO2_60lev_"+ens+"_sott_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_sott_"+time_stamp+".nc")
cdo.intlevel("876.1", input = cosmo_out+"CO2_60lev_"+ens+"_ssal_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_ssal_"+time_stamp+".nc")
cdo.intlevel("543.1", input = cosmo_out+"CO2_60lev_"+ens+"_taen_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_taen_"+time_stamp+".nc")
cdo.intlevel("615.8", input = cosmo_out+"CO2_60lev_"+ens+"_zhbr_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_zhbr_"+time_stamp+".nc")
cdo.intlevel("417.1", input = cosmo_out+"CO2_60lev_"+ens+"_zsch_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_zsch_"+time_stamp+".nc")
cdo.intlevel("408.8", input = cosmo_out+"CO2_60lev_"+ens+"_zue_"+time_stamp+".nc", output = cosmo_out+"modelled_"+ens+"_zue_"+time_stamp+".nc")
cdo.cat(input = cosmo_out+"modelled_"+ens+"_*_"+time_stamp+".nc ", output = cosmo_save+"model_"+ens+"_"+time_stamp+".nc")
logging.info('Extracting done for ens %s' % (ens))
else:
for l, ll in enumerate(hhl):
if float(station['alt']) < ll:
station['h1'] = hhl[l]
station['h2'] = hhl[l+1]
station['hidx1'] = l
station['hidx2'] = l+1
# The following line: we interpolate on the middle of alt - lower level
station['alt'] = float(station['alt']) - (float(station['alt']) - hhl[l+1])/2.
station['rlon'] = myrlon
station['rlat'] = myrlat
return rlat, rlon, dicts, path_in
def get_cosmo_data(self, dacycle, date_begin, dicts, rlat, rlon, path_in):
hours = ['12', '13', '14', '15']
qv_int = np.empty(shape=(60))
co2_bg_int = np.empty(shape=(60))
co2_gpp_int = np.empty(shape=(60))
co2_ra_int = np.empty(shape=(60))
co2_a_int = np.empty(shape=(60))
qv_int = np.empty(shape=(60))
co2_bg_daily = []
co2_gpp_daily = []
co2_ra_daily = []
co2_a_daily = []
co2_daily = []
for hrs in hours:
with Dataset(path_in+'lffd'+date_begin.strftime("%Y%m%d")+hrs+'.nc') as nc2:
qv = np.squeeze(nc2.variables['QV'][:])
co2_a = np.squeeze(nc2.variables['CO2_A'][:])
for station in dicts:
myrlat = station['rlat']
myrlon = station['rlon']
e = station['ensnum']
h1 = station['h1']
h2 = station['h2']
i1 = station['hidx1']
i2 = station['hidx2']
co2_bg = np.squeeze(nc2.variables['CO2_BG'+e][:])
co2_gpp = np.squeeze(nc2.variables['CO2_GPP'+e][:])
co2_ra = np.squeeze(nc2.variables['CO2_RA'+e][:])
for h in range(60):
qv_int[h] = interpolate.interpn((rlat,rlon), qv[h,:,:], [myrlat,myrlon], method='linear')
co2_bg_int[h] = interpolate.interpn((rlat,rlon), co2_bg[h,:,:], [myrlat,myrlon], method='linear')
co2_gpp_int[h] = interpolate.interpn((rlat,rlon), co2_gpp[h,:,:], [myrlat,myrlon], method='linear')
co2_ra_int[h] = interpolate.interpn((rlat,rlon), co2_ra[h,:,:], [myrlat,myrlon], method='linear')
co2_a_int[h] = interpolate.interpn((rlat,rlon), co2_a[h,:,:], [myrlat,myrlon], method='linear')
co2_bg1 = co2_bg_int[i1]
co2_bg2 = co2_bg_int[i2]
co2_gpp1 = co2_gpp_int[i1]
co2_gpp2 = co2_gpp_int[i2]
co2_ra1 = co2_ra_int[i1]
co2_ra2 = co2_ra_int[i2]
co2_a1 = co2_a_int[i1]
co2_a2 = co2_a_int[i2]
co2_a2 = co2_a_int[i2]
qv1 = qv_int[i1]
qv2 = qv_int[i2]
if h1 == h2:
co2_bg_final = co2_bg1
co2_gpp_final = co2_gpp1
co2_ra_final = co2_ra1
co2_a_final = co2_a1
qv_final = qv1
else:
co2_bg_final = co2_bg1 + (float(station['alt'])-h1)*(co2_bg2-co2_bg1)/(h2-h1)
co2_gpp_final = co2_gpp1 + (float(station['alt'])-h1)*(co2_gpp2-co2_gpp1)/(h2-h1)
co2_ra_final = co2_ra1 + (float(station['alt'])-h1)*(co2_ra2-co2_ra1)/(h2-h1)