Commit f7a938f2 authored by Woude, Auke van der's avatar Woude, Auke van der
Browse files

Added biosphere, multiple countries and biosphere

parent c0706f73
This diff is collapsed.
...@@ -14,12 +14,14 @@ import sys ...@@ -14,12 +14,14 @@ import sys
import logging import logging
import datetime as dtm import datetime as dtm
from string import strip
from numpy import array, logical_and from numpy import array, logical_and
import numpy as np import numpy as np
sys.path.append(os.getcwd()) sys.path.append(os.getcwd())
sys.path.append('../../') sys.path.append('../../')
from pylab import * #from pylab import *
from numpy import *
from scipy import *
from matplotlib.pylab import *
identifier = 'Rotterdam CO2 mole fractions' identifier = 'Rotterdam CO2 mole fractions'
version = '0.0' version = '0.0'
...@@ -29,7 +31,7 @@ import da.tools.io4 as io ...@@ -29,7 +31,7 @@ import da.tools.io4 as io
import da.tools.rc as rc import da.tools.rc as rc
################### Begin Class RdamObservations ################### ################### Begin Class RdamObservations ###################
class RdamObservations(Observations): class RINGOObservations(Observations):
""" an object that holds data + methods and attributes needed to manipulate mole fraction values """ """ an object that holds data + methods and attributes needed to manipulate mole fraction values """
def setup(self, dacycle): def setup(self, dacycle):
...@@ -62,28 +64,32 @@ class RdamObservations(Observations): ...@@ -62,28 +64,32 @@ class RdamObservations(Observations):
""" """
infile = os.path.join(self.obspack_dir, self.obspack_id) infile = os.path.join(self.obspack_dir, self.obspack_id)
logging.debug('infile = {}'.format(infile))
f = open(infile, 'r') f = open(infile, 'r')
lines = f.readlines() lines = f.readlines()
f.close() f.close()
ncfilelist = []
for line in lines:
if line.startswith('#'): continue # header
dum = line.split(",") ncfilelist = ['obsfiles/' + line.split(',')[1] for line in lines if not line[0] == '#']
ncfile = [dum[1]] ncfilelist = ncfilelist[:int(dacycle.dasystem['obs.input.nr'])]
# ncfilelist = []
if not ncfile[0] in ncfilelist: # for line in lines:
ncfilelist.extend(ncfile) # if line.startswith('#'): continue # header
#
# dum = line.split(",")
# ncfile = [dum[1]]
#
# if not ncfile[0] in ncfilelist:
# ncfilelist.extend(ncfile)
for ncfile in ncfilelist: for ncfile in ncfilelist:
infile = os.path.join(self.obspack_dir, ncfile + '.nc') infile = os.path.join(self.obspack_dir, ncfile + '.nc')
ncf = io.ct_read(infile, 'read') ncf = io.ct_read(infile, 'read')
idates = ncf.get_variable('Times') idates = ncf.get_variable('Times')
idates = [b''.join(d) for d in idates] idates = [''.join(d) for d in idates]
dates = array([dtm.datetime.strptime(d.decode(), '%Y-%m-%d_%H:%M:%S') for d in idates]) dates = array([dtm.datetime.strptime(d, '%Y-%m-%d_%H:%M:%S') for d in idates])
subselect = logical_and(dates >= self.startdate , dates <= self.enddate).nonzero()[0] subselect = logical_and(dates >= self.startdate , dates < self.enddate).nonzero()[0]
dates = dates.take(subselect, axis=0) dates = dates.take(subselect, axis=0)
datasetname = ncfile # use full name of dataset to propagate for clarity datasetname = ncfile # use full name of dataset to propagate for clarity
...@@ -120,7 +126,7 @@ class RdamObservations(Observations): ...@@ -120,7 +126,7 @@ class RdamObservations(Observations):
logging.info("Successfully read data from model sample file (%s)" % filename) logging.info("Successfully read data from model sample file (%s)" % filename)
obs_ids = self.getvalues('id').tolist() obs_ids = self.getvalues('id').tolist()
ids = list(map(int, ids)) obs_ids = list(map(int, obs_ids))
missing_samples = [] missing_samples = []
...@@ -135,7 +141,7 @@ class RdamObservations(Observations): ...@@ -135,7 +141,7 @@ class RdamObservations(Observations):
logging.warning('Model samples were found that did not match any ID in the observation list. Skipping them...') logging.warning('Model samples were found that did not match any ID in the observation list. Skipping them...')
logging.debug("Added %d simulated values to the Data list" % (len(ids) - len(missing_samples))) logging.debug("Added %d simulated values to the Data list" % (len(ids) - len(missing_samples)))
self.simulated_concs = simulated
def write_sample_coords(self, obsinputfile): def write_sample_coords(self, obsinputfile):
""" """
...@@ -323,7 +329,6 @@ class RdamObservations(Observations): ...@@ -323,7 +329,6 @@ class RdamObservations(Observations):
identifier = obs.code identifier = obs.code
# species, site, method, lab, datasetnr = identifier.split('_') # species, site, method, lab, datasetnr = identifier.split('_')
if identifier in site_info: if identifier in site_info:
if identifier in site_hourly: if identifier in site_hourly:
obs.samplingstrategy = 2 obs.samplingstrategy = 2
......
This diff is collapsed.
...@@ -128,9 +128,8 @@ def forward_pipeline(dacycle, platform, dasystem, samples, statevector, obsopera ...@@ -128,9 +128,8 @@ def forward_pipeline(dacycle, platform, dasystem, samples, statevector, obsopera
def analysis_pipeline(dacycle, platform, dasystem, samples, statevector): def analysis_pipeline(dacycle, platform, dasystem, samples, statevector):
""" Main entry point for analysis of ctdas results """ """ Main entry point for analysis of ctdas results """
from da.analysis.expand_fluxes import save_weekly_avg_1x1_data, save_weekly_avg_state_data, save_weekly_avg_tc_data, save_weekly_avg_ext_tc_data, save_weekly_avg_agg_data from da.analysis.expand_fluxes_ffdas import save_weekly_avg_1x1_data, save_weekly_avg_state_data, save_weekly_avg_tc_data, save_weekly_avg_ext_tc_data, save_weekly_avg_agg_data
from da.analysis.expand_molefractions import write_mole_fractions from da.analysis.expand_molefractions_ffdas import write_mole_fractions
from da.analysis.summarize_obs import summarize_obs
from da.analysis.time_avg_fluxes import time_avg from da.analysis.time_avg_fluxes import time_avg
logging.info(header + "Starting analysis" + footer) logging.info(header + "Starting analysis" + footer)
...@@ -158,16 +157,16 @@ def analysis_pipeline(dacycle, platform, dasystem, samples, statevector): ...@@ -158,16 +157,16 @@ def analysis_pipeline(dacycle, platform, dasystem, samples, statevector):
# save_weekly_avg_agg_data(dacycle,region_aggregate='olson_extended') # save_weekly_avg_agg_data(dacycle,region_aggregate='olson_extended')
# save_weekly_avg_agg_data(dacycle,region_aggregate='country') # save_weekly_avg_agg_data(dacycle,region_aggregate='country')
logging.info(header + "Starting monthly and yearly averages" + footer) #logging.info(header + "Starting monthly and yearly averages" + footer)
time_avg(dacycle,'flux1x1') #time_avg(dacycle,'flux1x1')
# time_avg(dacycle,'transcom') # time_avg(dacycle,'transcom')
# time_avg(dacycle,'transcom_extended') # time_avg(dacycle,'transcom_extended')
# time_avg(dacycle,'olson') # time_avg(dacycle,'olson')
# time_avg(dacycle,'olson_extended') # time_avg(dacycle,'olson_extended')
# time_avg(dacycle,'country') # time_avg(dacycle,'country')
logging.info(header + "Finished analysis" + footer) #logging.info(header + "Finished analysis" + footer)
def archive_pipeline(dacycle, platform, dasystem): def archive_pipeline(dacycle, platform, dasystem):
""" Main entry point for archiving of output from one disk/system to another """ """ Main entry point for archiving of output from one disk/system to another """
...@@ -233,17 +232,17 @@ def prepare_state(dacycle, dasystem, statevector,samples, obsoperator, emismodel ...@@ -233,17 +232,17 @@ def prepare_state(dacycle, dasystem, statevector,samples, obsoperator, emismodel
# Fill each week from n=1 to n=nlag with a new ensemble # Fill each week from n=1 to n=nlag with a new ensemble
for n in range(statevector.nlag): for lag in range(statevector.nlag):
date = dacycle['time.start'] + datetime.timedelta(days=(n + 0.5) * int(dacycle['time.cycle'])) date = dacycle['time.start'] + datetime.timedelta(days=(lag + 0.5) * int(dacycle['time.cycle']))
cov = statevector.get_covariance(date, dacycle) cov = statevector.get_covariance(date, dacycle)
statevector.make_new_ensemble(dacycle, n, cov) statevector.make_new_ensemble(lag, cov)
# Prepare emissions (spatial and temporal distribution) if emisflag is set to 1 # Prepare emissions (spatial and temporal distribution) if emisflag is set to 1
if dacycle.dasystem['run.emisflag']=='1': if dacycle.dasystem['run.emisflag']=='1':
emismodel.get_emis(dacycle,psdo=1) emismodel.get_emis(dacycle, samples, do_pseudo=1)
# Prepare new pseudo-observations if obsflag is set to 1 # Prepare new pseudo-observations if obsflag is set to 1
if dacycle.dasystem['run.obsflag']=='1': if dacycle.dasystem['run.obsflag']=='1':
obsoperator.run_forecast_model(dacycle,psdo=1,adv=0) obsoperator.run_forecast_model(dacycle,do_pseudo=1,adv=0)
else: else:
# Read the statevector data from file # Read the statevector data from file
...@@ -325,12 +324,11 @@ def sample_step(dacycle, samples, statevector, obsoperator, emismodel, lag, adva ...@@ -325,12 +324,11 @@ def sample_step(dacycle, samples, statevector, obsoperator, emismodel, lag, adva
# Run the observation operator, store data in different file for advance # Run the observation operator, store data in different file for advance
if dacycle.dasystem['run.emisflagens']=='1': if dacycle.dasystem['run.emisflagens']=='1':
emismodel.get_emis(dacycle,psdo=0) emismodel.get_emis(dacycle, samples, do_pseudo=0)
if not advance: if not advance:
obsoperator.run_forecast_model(dacycle,psdo=0,adv=0) obsoperator.run_forecast_model(dacycle,do_pseudo=0,adv=0)
else: else:
obsoperator.run_forecast_model(dacycle,psdo=0,adv=1) obsoperator.run_forecast_model(dacycle,do_pseudo=0,adv=1)
# Read forecast model samples that were written to NetCDF files by each member. Add them to the exisiting # Read forecast model samples that were written to NetCDF files by each member. Add them to the exisiting
# Observation object for each sample loop. This data fill be written to file in the output folder for each sample cycle. # Observation object for each sample loop. This data fill be written to file in the output folder for each sample cycle.
...@@ -342,7 +340,6 @@ def sample_step(dacycle, samples, statevector, obsoperator, emismodel, lag, adva ...@@ -342,7 +340,6 @@ def sample_step(dacycle, samples, statevector, obsoperator, emismodel, lag, adva
if os.path.exists(sampling_coords_file): if os.path.exists(sampling_coords_file):
samples.add_simulations(obsoperator.simulated_file) samples.add_simulations(obsoperator.simulated_file)
else: logging.warning("No simulations added, because input file does not exist (no samples found in obspack)") else: logging.warning("No simulations added, because input file does not exist (no samples found in obspack)")
# Now write a small file that holds for each observation a number of values that we need in postprocessing # Now write a small file that holds for each observation a number of values that we need in postprocessing
#filename = samples.write_sample_coords() #filename = samples.write_sample_coords()
......
!!! Info for the CarbonTracker data assimilation system !!! Info for the CarbonTracker data assimilation system
datadir : /home/awoude/ffdas/test/Data datadir : /home/awoude/ffdas/RINGO/Data
! list of all observation sites ! list of all observation sites
obs.input.id : obsfiles.csv obs.input.id : obsfiles.csv
! number of observation sites included; number of species included and to be used in inversion ! number of observation sites included; number of species included and to be used in inversion
obs.input.nr : 7 obs.input.nr : 2
obs.spec.nr : 4 obs.spec.nr : 1
obs.spec.name : co2
! number of emission categories defined in the emission model ! number of emission categories defined in the emission model
obs.cat.nr : 14 obs.cat.nr : 14
! For Rdam obs ! For Rdam obs
obs.sites.rc : ${datadir}/sites_weights.rc obs.sites.rc : ${datadir}/sites_weights.rc
obs.sites.times : 12; 16
! number of parameters ! number of parameters
nparameters : 44 nparameters : 50
! set fixed seed for random number generator, or use 0 if you want to use any random seed ! set fixed seed for random number generator, or use 0 if you want to use any random seed
random.seed : 4385 random.seed : 4385
!file with prior estimate of scaling factors (statevector) and covariances !file with prior estimate of scaling factors (statevector) and covariances
emis.pparam : param_values.nc emis.pparam : param_values.nc
ff.covariance : covariances.nc ff.covariance : covariances2.nc
!file with emission model parameter values !file with emission model parameter values
emis.paramfile : emis_parameters.csv emis.paramfiles : emission_factors
emis.ncountries : 10
countries : AUS; BEL; CZE; FRA; DEU; LUX; NED; POL; CHE; GBR
! switch (1=on/0=off) and input data for background CO2 and CO concentrations ! switch (1=on/0=off) and input data for background CO2 and CO concentrations
obs.bgswitch : 1 obs.bgswitch : 1
...@@ -28,8 +32,11 @@ obs.background : ${datadir}/background.nc ...@@ -28,8 +32,11 @@ obs.background : ${datadir}/background.nc
! input data for emission model ! input data for emission model
emis.input.spatial : spatial_data.nc emis.input.spatial : spatial_data.nc
emis.input.tempobs : temporal_data.nc emis.input.tempobs : time_profiles_stations.nc
emis.input.tempprior : temporal_data.nc emis.input.tempprior : time_profiles_stations.nc
! Area of the gridboxes
area.file : /home/awoude/ffdas/RINGO/Data/area.nc
! overwrite existing prior/ensemble emission files + pseudo-data (0: keep existing files; 1: create new files) ! overwrite existing prior/ensemble emission files + pseudo-data (0: keep existing files; 1: create new files)
run.emisflag : 0 run.emisflag : 0
...@@ -37,10 +44,10 @@ run.emisflagens : 1 ...@@ -37,10 +44,10 @@ run.emisflagens : 1
run.obsflag : 0 run.obsflag : 0
! back trajectory time of STILT footprints, also applied to OPS (in hours) ! back trajectory time of STILT footprints, also applied to OPS (in hours)
run.backtime : 6 run.backtime : 24
! choose propagation scheme: ! choose propagation scheme:
! 1: no propagation, start each cycle with the same prior parameter values and covariance matrix ! 1: no propagation, start each cycle with the same prior parameter values and covariance matrix
! 2: propagation of optimized parameter values, but not of the covariance matrix ! 2: propagation of optimized parameter values, but not of the covariance matrix
! 3: propagation of both optimized parameter values and covariance matrix ! 3: propagation of both optimized parameter values and covariance matrix
run.propscheme : 3 run.propscheme : 2
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment