Commit 0b249e54 authored by Roelofsen, Hans's avatar Roelofsen, Hans
Browse files

adding POSCH twice

parent 92cfb8ce
......@@ -11,7 +11,8 @@ param_header_src = r'W:\PROJECTS\Doren19\a_brondata\EVA\delivery_20201118\EVA_Do
param_sp_src = r'W:\PROJECTS\Doren19\a_brondata\EVA\delivery_201909\EVA_Doren_species.csv'
param_aoi_src = r'W:\PROJECTS\Doren19\a_brondata\AOI\ne_50m_cntrs_AOI_diss_fin.shp'
param_dem_src = r'W:\PROJECTS\Doren19\a_brondata\covariables\DEM\DTM_3035.tif'
param_posch = r'W:\PROJECTS\Doren19\a_brondata\POSCH_dep\20201012delivery'
param_posch_single = r'W:\PROJECTS\Doren19\a_brondata\POSCH_dep\20200401delivery\v2'
param_posch_diff = r'W:\PROJECTS\Doren19\a_brondata\POSCH_dep\20201012delivery'
cv_soil_dir = r'w:\PROJECTS\Doren19\a_brondata\covariables\soil\b_processed'
cv_soil_src = 'WRBLEV1_laea.tif'
cv_cntr_dir = r'w:\PROJECTS\Doren19\a_brondata\covariables\countries'
......@@ -20,7 +21,7 @@ cv_precp_dir = r'w:\PROJECTS\Doren19\a_brondata\covariables\EObs\2_compiled'
cv_precp_src = "EObs_v200e_rr_5yrmean"
cv_temp_dir = cv_precp_dir
cv_temp_src = "EObs_v200e_tg_5yrmean"
sp_req_src = r'c:\Users\roelo008\Wageningen University & Research\DOREN - General\DOREN-2020-11-30.xlsx'
sp_req_src = r'c:\Users\roelo008\Wageningen University & Research\DOREN - General\DOREN-2020-12-02.xlsx'
sp_req_sheet, column, skip = 'Soorten', 'wetenschappelijke naam', 1
# sp_req_src = r'c:\Users\roelo008\Wageningen University & Research\DOREN - General\2020-09-17 uniek soorten per habitat (2).xlsx'
......@@ -37,7 +38,8 @@ doren.add_yearly_covar(covar_dir=cv_precp_dir, covar_src_basename=cv_precp_src,
doren.add_yearly_covar(covar_dir=cv_temp_dir, covar_src_basename=cv_temp_src, covar_name='five_yearly_temp')
doren.add_eunis()
doren.add_posch(posch_src_dir=param_posch)
doren.add_posch_single(posch_src_dir=param_posch_single)
doren.add_posch_differentiated(posch_src_dir=param_posch_diff)
doren.test()
'''
......
......@@ -20,4 +20,4 @@ echo $SLURM_ARRAY_TASK_ID
# Run
cd /home/WUR/roelo008/projs/doren_2019
python run_species.py $SLURM_ARRAY_TASK_ID 50 doren_20201202.pkl
\ No newline at end of file
python run_species.py $SLURM_ARRAY_TASK_ID 50 doren_20201207.pkl
\ No newline at end of file
......@@ -318,7 +318,57 @@ class Doren:
self.report += 'Union between requirements {0} gives {1} rows that DO NOT meet requirements. ' \
'Remaining: {2}\n\n'.format(', '.join(requirements), len(drop_rows), self.status['n_plots'])
def add_posch(self, posch_src_dir):
def add_posch_single(self, posch_src_dir):
# posch_src_dir = r'\\wur\dfs-root\PROJECTS\Doren19\a_brondata\POSCH_dep\20200401delivery'
try:
nh3 = pd.read_csv(os.path.join(posch_src_dir, 'NH3-20200505.csv'), sep=',', comment='!',
index_col='plot_obs_id')
nox = pd.read_csv(os.path.join(posch_src_dir, 'NOx-20200505.csv'), sep=',', comment='!',
index_col='plot_obs_id')
except OSError:
print('Ndeposition data not found in {0}'.format(posch_src_dir))
raise
# rename columns to proper format
yr_cols = [x for x in list(nh3) if x.isdigit()]
nh3.rename(columns=dict(zip(yr_cols, ['year_{0}'.format(y) for y in yr_cols])), inplace=True)
nox.rename(columns=dict(zip(yr_cols, ['year_{0}'.format(y) for y in yr_cols])), inplace=True)
# NH3
mapper = self.eva.assign(year=('year_' + self.eva.date_of_recording.astype(int).astype(str)),
plot_obs_id=self.eva.index)
c1 = mapper['plot_obs_id'].isin(nh3.index)
c2 = mapper['year'].isin(nh3.columns)
mapper = mapper.loc[c1 & c2]
self.eva.loc[c1 & c2, 'nh3_mg_m2'] = nh3.lookup(mapper['plot_obs_id'], mapper['year'])
# NOx
mapper = self.eva.assign(year=('year_' + self.eva.date_of_recording.astype(int).astype(str)),
plot_obs_id=self.eva.index)
c1 = mapper['plot_obs_id'].isin(nox.index)
c2 = mapper['year'].isin(nox.columns)
mapper = mapper.loc[c1 & c2]
self.eva.loc[c1 & c2, 'nox_mg_m2'] = nox.lookup(mapper['plot_obs_id'], mapper['year'])
# Drop NAs
self.eva.dropna(axis=0, subset=['nh3_mg_m2', 'nox_mg_m2'], how='any', inplace=True)
self.update_status()
# Calculate totals
self.eva['totN_mg_m2'] = self.eva.loc[:, ["nh3_mg_m2", "nox_mg_m2"]].sum(axis=1)
self.eva['totN_kg_ha'] = self.eva.loc[:, 'totN_mg_m2'].divide(100)
self.eva['totN_kmol_ha'] = self.eva.loc[:, 'totN_kg_ha'].divide(14)
# Reporting
self.update_status(covar=['totN_kmol_ha'])
msg = '\nAdded NDep Single from POSCH: {0} rows remaining.\n\n'.format(self.eva.shape[0])
self.report += msg
if self.verbose:
print(msg)
def add_posch_differentiated(self, posch_src_dir):
'''
Add N deposition data sourced from Max POSCH. Update November 2020 to differtiate between Ndep in Forests (f)
and Open vegetation (v). Add three flavours to self.eva: NDep(forest), NDep(Open), NDep(Forest/Open)(afh van structuurtype)
......@@ -355,33 +405,42 @@ class Doren:
nox_v.rename(columns=dict(zip(yr_cols, ['year_{0}'.format(y) for y in yr_cols])), inplace=True)
# NH3 - Forest to all plots
mapper = self.eva.assign(year=('year_' + self.eva.date_of_recording.astype(int).astype(str)), plot_obs_id=self.eva.index)
mapper = self.eva.assign(year=('year_' + self.eva.date_of_recording.astype(int).astype(str)),
plot_obs_id=self.eva.index)
c1 = mapper['plot_obs_id'].isin(nh3_f.index)
c2 = mapper['year'].isin(nh3_f.columns)
mapper = mapper.loc[c1 & c2]
self.eva.loc[c1 & c2, 'nh3_mg_m2_f'] = nh3_f.lookup(mapper['plot_obs_id'], mapper['year'])
# NH3 - Open to all plots
mapper = self.eva.assign(year=('year_' + self.eva.date_of_recording.astype(int).astype(str)), plot_obs_id=self.eva.index)
mapper = self.eva.assign(year=('year_' + self.eva.date_of_recording.astype(int).astype(str)),
plot_obs_id=self.eva.index)
c1 = mapper['plot_obs_id'].isin(nh3_v.index)
c2 = mapper['year'].isin(nh3_v.columns)
mapper = mapper.loc[c1 & c2]
self.eva.loc[c1 & c2, 'nh3_mg_m2_v'] = nh3_v.lookup(mapper['plot_obs_id'], mapper['year'])
# NOx - Forest to all plots
mapper = self.eva.assign(year=('year_' + self.eva.date_of_recording.astype(int).astype(str)), plot_obs_id=self.eva.index)
mapper = self.eva.assign(year=('year_' + self.eva.date_of_recording.astype(int).astype(str)),
plot_obs_id=self.eva.index)
c1 = mapper['plot_obs_id'].isin(nox_f.index)
c2 = mapper['year'].isin(nox_f.columns)
mapper = mapper.loc[c1 & c2]
self.eva.loc[c1 & c2, 'nox_mg_m2_f'] = nox_f.lookup(mapper['plot_obs_id'], mapper['year'])
# NOx - Open to all plots
mapper = self.eva.assign(year=('year_' + self.eva.date_of_recording.astype(int).astype(str)), plot_obs_id=self.eva.index)
mapper = self.eva.assign(year=('year_' + self.eva.date_of_recording.astype(int).astype(str)),
plot_obs_id=self.eva.index)
c1 = mapper['plot_obs_id'].isin(nox_v.index)
c2 = mapper['year'].isin(nox_v.columns)
mapper = mapper.loc[c1 & c2]
self.eva.loc[c1 & c2, 'nox_mg_m2_v'] = nox_v.lookup(mapper['plot_obs_id'], mapper['year'])
# Drop NAs
self.eva.dropna(axis=0, subset=['nh3_mg_m2_f', 'nh3_mg_m2_v', 'nox_mg_m2_f', 'nox_mg_m2_v'], how='any',
inplace=True)
self.update_status()
# Total NDep Forest plots
self.eva['totN_mg_m2_f'] = self.eva.loc[:, ["nh3_mg_m2_f", "nox_mg_m2_f"]].sum(axis=1)
self.eva['totN_kg_ha_f'] = self.eva.loc[:, 'totN_mg_m2_f'].divide(100)
......@@ -398,18 +457,15 @@ class Doren:
v = self.eva.loc[self.eva.hooglaag == 'laag'].index # Open vegetation
x = self.eva.index.difference(f.union(v)) # Anders/niet bekend
self.eva.loc[f, 'totN_kmol_ha'] = self.eva.loc[f, 'totN_kmol_ha_f']
self.eva.loc[v, 'totN_kmol_ha'] = self.eva.loc[v, 'totN_kmol_ha_v']
self.eva.loc[x, 'totN_kmol_ha'] = self.eva.loc[x, 'totN_kmol_ha_v'] # Gebruik Open Veg waneer onduidelijk
self.eva.loc[f, 'totN_kmol_ha_vf'] = self.eva.loc[f, 'totN_kmol_ha_f']
self.eva.loc[v, 'totN_kmol_ha_vf'] = self.eva.loc[v, 'totN_kmol_ha_v']
self.eva.loc[x, 'totN_kmol_ha_vf'] = self.eva.loc[x, 'totN_kmol_ha_v'] # Gebruik Open Veg waneer onduidelijk
else:
raise Exception('Cannot assign NDep data differenntiated to structuurtype. ')
# Drop NAs
self.eva.dropna(axis=0, subset=['totN_kmol_ha'], how='any', inplace=True)
self.update_status(covar=['totN_kmol_ha'])
msg = '\nAdded NDep from POSCH: {0} rows remaining.\n\n'.format(self.eva.shape[0])
self.update_status(covar=['totN_kmol_ha_vf'])
msg = '\nAdded NDep from POSCH with distinction open-forest: {0} rows remaining.\n\n'.format(self.eva.shape[0])
self.report += msg
if self.verbose:
print(msg)
......@@ -838,7 +894,7 @@ class Doren:
# Dedicated directory for the PG output and output filename
# TODO: replace non-ascii characters
pg_out_name = '{0}_NDep_Nearest'.format(self.sel_species_ascii)
pg_out_name = '{0}_NDep_Nearest'.format(self.sel_species_ascii.replace(' ', '_'))
pg_dir = os.path.join(out_dir, 'pg_input')
if not os.path.isdir(pg_dir):
os.mkdir(pg_dir)
......@@ -888,7 +944,7 @@ class Doren:
if not os.path.isdir(pg_dir):
os.mkdir(pg_dir)
out_cols = ['plot_obs_id', 'totN_kmol_ha', 'soil_type_label', 'country_label', 'five_yearly_precip',
out_cols = ['plot_obs_id', 'totN_kmol_ha_v', 'totN_kmol_ha_f', 'totN_kmol_ha_vf', 'totN_kmol_ha', 'soil_type_label', 'country_label', 'five_yearly_precip',
'five_yearly_temp', 'eunis_code', 'eunis_src', 'structuurtype', 'hooglaag']
for col in out_cols:
assert hasattr(self.eva, col), 'Cannot write due to missing column {}'.format(col)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment