a2_merge_POSCHdep.py 3.77 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
"""
Script for grabbing the NDep data from MAX POSCH and adding to the EVA header data
Hans Roelofsen, 01-04-2020

New colums to EVA headers are:
1) nh3 in mg/m2
2) nox in mg/m2
3) totale Ndep in mg/m2 (nh3 + nox)
4) totale Ndep in kg/ha
5) totale NDep in molN/ha (N [kg/ha] / 14)

EVA data and MAX POSCH data contain identical rows (EVA veg) plots. Use year column from Max Posch corresponding to
plot year.

"""

import pandas as pd
import datetime
import os
import pickle

from utils import doren as do

'''Source data'''
roelo008's avatar
roelo008 committed
25
26
eva_compiled_dir = r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\b_compiled_data\1_EVA'
eva_compiled_in = 'eva_header_selection_20200409-1437.pkl'
27
28
29
30
31
32

ndp_compiled_dir = r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\a_brondata\POSCH_dep'
nh3_in = 'NH3-20200331.csv'
nox_in = 'NOx-20200331.csv'

'''Output data'''
roelo008's avatar
roelo008 committed
33
out_dir = r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\b_compiled_data\2_EVA_Posch'
34
35
36
37
38
39
40
41
42
43
44
45
46
datestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M")
basename = 'eva_headers_w_NDep_MAX_{0}'.format(datestamp)

eva = pd.read_pickle(os.path.join(eva_compiled_dir, eva_compiled_in))
eva['year'] = eva.apply(lambda row: do.int_year(row.date_of_recording), axis=1)

nh3 = pd.read_csv(os.path.join(ndp_compiled_dir, nh3_in), sep=',', comment='!', index_col='plot_obs_id')
nox = pd.read_csv(os.path.join(ndp_compiled_dir, nox_in), sep=',', comment='!', index_col='plot_obs_id')
yr_cols = [x for x in list(nh3) if x.isdigit()]
nh3.rename(columns=dict(zip(yr_cols, ['year_{0}'.format(y) for y in yr_cols])), inplace=True)
nox.rename(columns=dict(zip(yr_cols, ['year_{0}'.format(y) for y in yr_cols])), inplace=True)

'''Add columns 1 and 2
47
48
https://stackoverflow.com/questions/60976021/pandas-select-from-column-with-index-corresponding-to
-values-in-another-column/
49
50
'''
#NH3
roelo008's avatar
roelo008 committed
51
mapper = eva.assign(year=('year_' + eva['year'].astype(str)), plot_obs_id=eva.index)
52
53
54
55
56
57
58
c2 = mapper['plot_obs_id'].isin(nh3.index)
c1 = mapper['year'].isin(nh3.columns)
mapper = mapper.loc[c1 & c2]
eva.loc[c2 & c1, 'nh3_mg/m2'] = nh3.lookup(mapper['plot_obs_id'], mapper['year'])
eva['nh3_mg/m2'] = eva['nh3_mg/m2'].fillna(-99)

# NOX
roelo008's avatar
roelo008 committed
59
mapper = eva.assign(year=('year_' + eva['year'].astype(str)), plot_obs_id=eva.index)
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
c2 = mapper['plot_obs_id'].isin(nox.index)
c1 = mapper['year'].isin(nox.columns)
mapper = mapper.loc[c1 & c2]
eva.loc[c2 & c1, 'nox_mg/m2'] = nox.lookup(mapper['plot_obs_id'], mapper['year'])
eva['nox_mg/m2'] = eva['nox_mg/m2'].fillna(-99)

'''Calculate total N in various units'''
eva['totN_mg/m2'] = eva.loc[:, ["nh3_mg/m2", "nox_mg/m2"]].sum(axis=1)
eva['totN_kg/ha'] = eva.loc[:, 'totN_mg/m2'].divide(100)
eva['totN_mol/ha'] = eva.loc[:, 'totN_kg/ha'].divide(14)

'''Save to files'''
# Text metadata
file_desc = 'File {0} contains EVA plot header selection based on {1}, with N Dep data from MAX POSCH (source 1: {2}' \
          ', source 2: {3}) added as additional columns.'.format(basename, eva_compiled_in, nh3_in, nox_in)
75
source = 'See git for source script: https://git.wur.nl/roelo008/doren_2019'
76
77
78
footer = '\nMade with Python 3.5 using Pandas by Hans Roelofsen, WEnR team B&B, dated {0}'.format(datestamp)
with open(os.path.join(out_dir, '{0}.txt'.format(basename)), 'w') as f:
    f.write(file_desc)
79
    f.write(source)
80
81
82
    f.write(footer)

# to CSV
83
eva.drop(labels=['plot_coordinates_wgs84', 'plot_coordinates_3035', "plot_pnt_wgs84"], axis=1).\
roelo008's avatar
roelo008 committed
84
    to_csv(os.path.join(out_dir, '{0}.csv'.format(basename)), sep=';')
85
86
87
88
89
90
91
92
93
"""NOTE: coordinate tuples are stored as string and cannot be recovered as tuple! """

# To pickle
with open(os.path.join(out_dir, '{0}.pkl'.format(basename)), 'wb') as handle:
    pickle.dump(eva, handle)

# To Shapefile
eva.drop(labels=['plot_coordinates_wgs84', 'plot_coordinates_3035'], axis=1) \
       .to_file(os.path.join(out_dir, '{0}.shp'.format(basename)))