Commit d9102dc2 authored by roelo008's avatar roelo008
Browse files

Adding new single file output method plus additional verification of shapefile covars

parent 0954d491
......@@ -11,7 +11,7 @@ param_posch = r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\a_bro
cv_soil_dir = r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\a_brondata\covariables\soil\b_processed'
cv_soil_src = 'WRBLEV1_laea.tif'
cv_cntr_dir = r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\geodata\countries'
cv_cntr_src = 'ne_50m_cntrs_sel_buff_diss_3035.shp'
cv_cntr_src = 'ne_50m_cntrs_sel_buff_diss_2_3035.shp'
cv_precp_dir = r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\a_brondata\covariables\EObs\2_compiled'
cv_precp_src = "EObs_v200e_rr_5yrmean"
cv_temp_dir = r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\a_brondata\covariables\EObs\2_compiled'
......@@ -20,11 +20,12 @@ sp_req_src = r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\z_scra
doren = dc.Doren(header_src= param_header_src, sp_src=param_sp_src)
doren.initiate()
doren.rename(old_name='Oxycoccus palustris', new_name='Vaccinium oxycoccos')
doren.apply_requirements('req1', 'req2', 'req3', 'req4', 'req8', 'req9', 'req10', aoi_src=param_aoi_src,
dem_src=param_dem_src)
doren.add_covar(covar_dir=cv_cntr_dir, covar_src=cv_cntr_src, covar_name='country', raster=False, column='SOVEREIGNT')
doren.add_posch(posch_src_dir=param_posch)
doren.add_covar(covar_dir=cv_soil_dir, covar_src=cv_soil_src, covar_name='soil_type', nominal=True)
doren.add_covar(covar_dir=cv_cntr_dir, covar_src=cv_cntr_src, covar_name='country', raster=False, column='SOVEREIGNT')
doren.add_yearly_covar(covar_dir=cv_precp_dir, covar_src_basename=cv_precp_src, covar_name='5_yearly_precip')
doren.add_yearly_covar(covar_dir=cv_temp_dir, covar_src_basename=cv_temp_src, covar_name='5_yearly_temp')
doren.get_requested_species(src=sp_req_src)
......@@ -32,7 +33,7 @@ doren.get_requested_species(src=sp_req_src)
doren.write_stuff('species_list')
doren.write_stuff('typische_soorten')
doren.write_stuff('report')
# doren.write_stuff('headers_shp')
doren.write_stuff('headers_shp')
with open(r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\b_compiled_data\a_pkl\doren_{}.pkl'.
format(doren.timestamp), 'wb') as handle:
......
......@@ -35,9 +35,7 @@ for i, j in enumerate(sel, start=1):
print(' doing species {0} of {1}: {2}'.format(i, len(sel), sp))
doren.select_plts_w_species(species_name=sp)
doren.filter_by_buffer_around_positive_plots(buffer_size=100000)
doren.write_stuff('species_PG')
doren.write_stuff('NDep_PG', covars=True)
doren.write_stuff('NDep_PG')
doren.write_stuff('species_single_file')
except OSError:
continue
......@@ -213,7 +213,7 @@ def lon_lat_2_east_north(lon_lat):
(east, north) = pyproj.transform(in_proj, out_proj, lon, lat)
return shapely.geometry.Point((east,north))
return shapely.geometry.Point((east, north))
def generate_square_id(east_north, size):
......
......@@ -152,6 +152,28 @@ class Doren:
if self.verbose:
print(msg1, msg2)
def rename(self, old_name, new_name):
"""
Rename a species in self.sp and self.species
:param old_name: species name to be renamed
:param new_name: new species name
:return: updated self.sp, self.species
"""
assert hasattr(self, 'spec'), 'Initiate DOREN object first before messing with plant names'
pre_count = self.spec.loc[self.spec.species_name_hdr == old_name].shape[0]
self.spec.species_name_hdr.replace(to_replace=old_name, value=new_name, inplace=True)
self.update_status()
post_count_old = self.spec.loc[self.spec.species_name_hdr == old_name].shape[0]
post_count_new = self.spec.loc[self.spec.species_name_hdr == new_name].shape[0]
# if (pre_count == post_count_new) and (post_count_old == 0):
# msg = ' replaced {0} occurences of {1} with {2}.\n'.format(pre_count, old_name, new_name)
# if self.verbose:
# print(msg)
# self.report += msg
# else:
# raise TypeError('Something went wrong with finding and replacing species name {0}'.format(old_name))
def apply_requirements(self, *reqs, **kwargs):
"""
Filter down EVA headers by applying one or more requirements
......@@ -189,6 +211,8 @@ class Doren:
possible_requirements['req2'] = self.eva.loc[self.eva.longitude.isna(), :].index
if 'req3' in requirements:
if aoi_src is None:
raise OSError('Please provide AOI source shapefile.')
possible_requirements['req3'] = \
self.eva.index.difference(gp.sjoin(left_df=self.eva, right_df=do.get_aoi(aoi_src), how='inner',
op='within', lsuffix='eva_', rsuffix='aoi_').index)
......@@ -306,10 +330,22 @@ class Doren:
# Get the values from a column in a polygon shapefile
column = kwargs.get('column', Exception('Provide column keyword argument'))
covar_gdf = gp.read_file(os.path.join(covar_dir, covar_src))
joined = gp.sjoin(left_df=self.eva, right_df=covar_gdf.loc[:, [column, 'geometry']], how='left',
op='within', lsuffix='eva_', rsuffix='covar_')
assert joined.shape[0] == self.eva.shape[0], ' oh ffs'
cov_df = pd.DataFrame(data={'label': joined.loc[:, column].values}, index=self.eva.index)
if not all(covar_gdf.geometry.is_valid) or any(covar_gdf.geometry.isna()):
# TODO: Identify invalid features
warn = ' !! invalid geometries detected for {0}'.format(covar_src)
raise Exception(warn)
if any(covar_gdf.loc[:, column].isna()):
warn = ' !! na values detected for column {}'.format(column)
raise Exception(warn)
try:
joined = gp.sjoin(left_df=self.eva, right_df=covar_gdf, how='left', op='within', lsuffix='eva_',
rsuffix='covar_')
except ValueError:
joined = gp.sjoin(left_df=self.eva.sample(self.eva.shape[0]), right_df=covar_gdf, how='left',
op='within', lsuffix='eva_', rsuffix='covar_')
# To catch ValueError: Null geometry supports no operations
assert joined.shape[0] == self.eva.shape[0], ' spatial join shape does not match self.eva shape'
cov_df = pd.DataFrame(data={'label': joined.loc[:, column].values}, index=joined.index)
# drop NA, rename columns to reflect covariable name
cov_df.dropna(axis=0, how='any', inplace=True)
......@@ -436,7 +472,7 @@ class Doren:
csv_out_dir = os.path.join(out_dir, 'csv')
if not os.path.isdir(csv_out_dir):
os.mkdir(csv_out_dir)
req_out_name = '{0}_{1}_typische_soorten.csv'.format(self.basename, self.timestamp)
req_out_name = '{0}_{1}_typische_soorten.csv'.format(self.basename, self.timestamp)
self.request_df.to_csv(os.path.join(csv_out_dir, req_out_name), sep=',', index=False)
......@@ -539,6 +575,39 @@ class Doren:
self.report += 'Written FORTRAN input to files: {0}\n\n'.format(os.path.join(pg_dir, pg_out_name))
elif what == 'species_single_file':
"""Single file for union self.positive_plots, self.nearby_plots formatted as:
PlotObservationID,totN_mol_ha,soil_type_vals,soil_type_label,country_label,5_yearly_precip,5_yearly_temp,response
87523,0.5481642857142858,11.0,jong,Spain,4.240024717242157,12.729687707377398,0
87524,0.4981642857142857,11.0,jong,Spain,3.376466375059885,11.764382274150906,0
See e-mail Paul Goedhart, 25-06-2020
"""
# check if all required data is present
if self.sel_species is None or 'totN_mol_ha' not in self.eva.columns:
print('Cannot report for PG')
return None
# Dedicated directory for the PG output and output filename
pg_out_name = '{0}_NDep_{1}'.format(self.sel_species.replace(' ', '_'), str(int(self.buffer_size / 1000)))
pg_dir = os.path.join(out_dir, 'pg_input')
if not os.path.isdir(pg_dir):
os.mkdir(pg_dir)
# Compile dataframe for writing to file
out_cols = ['plot_id', 'totN_mol_ha', 'soil_type_label', 'country_label', '5_yearly_precip', '5_yearly_temp']
out_df = self.eva.loc[self.positive_plots.union(self.nearby_plots), out_cols]
out_df.loc[self.positive_plots, 'respons'] = np.uint8(1)
out_df.loc[self.nearby_plots, 'respons'] = np.uint8(0)
# Write to file and update report
out_df.astype({'respons': 'uint8'}).to_csv(os.path.join(pg_dir, '{}.csv'.format(pg_out_name)), sep=',',
index=False, header=True)
self.report += 'Written single file output for species {0} to file: ' \
'{1}'.format(self.sel_species, os.path.join(pg_dir, pg_out_name))
elif what == 'headers_shp':
shp_out_dir = os.path.join(out_dir, 'shp')
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment