Commit 24cc4c5f authored by roelo008's avatar roelo008
Browse files

prep 4 batch species processing

parent bb8897a1
import datetime
import pickle
from utils import doren_classes as dc
doren = dc.Doren(header_src=r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\a_brondata\EVA\EVA_Doren_header.csv',
sp_src=r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\a_brondata\EVA\EVA_Doren_species.csv')
doren.initiate()
doren.apply_requirements('req1', 'req2', 'req3', 'req4', 'req8', 'req9', 'req10',
aoi_src=r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\geodata\eva_aoi_fin_3035.shp',
dem_src=r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\a_brondata\covariables\DEM\DTM_3035.tif')
doren.add_posch(posch_src_dir=r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\a_brondata\POSCH_dep\v2')
doren.add_covar(covar_dir=r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\a_brondata\covariables\soil\b_processed',
covar_src='WRBLEV1_laea.tif', covar_name='soil_type', nominal=True)
doren.add_covar(covar_dir=r'c:\Users\roelo008\OneDrive - WageningenUR\b_geodata\eur_cntrs_3035',
covar_src='esri_eur_cntrs_3035.shp', covar_name='country', raster=False, column='CNTRYNAME',
keep_all=True)
doren.add_yearly_covar(covar_dir=r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\a_brondata\covariables\EObs\2_compiled',
covar_src_basename="EObs_v200e_rr_5yrmean", covar_name='5_yearly_precip')
doren.add_yearly_covar(covar_dir=r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\a_brondata\covariables\EObs\2_compiled',
covar_src_basename="EObs_v200e_tg_5yrmean", covar_name='5_yearly_temp')
doren.get_requested_species(src=r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\z_scratch\soortenlijst.csv')
with open(r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\b_compiled_data\a_pkl\doren_{}.pkl'.
format(doren.timestamp), 'wb') as handle:
pickle.dump(doren, handle, protocol=pickle.HIGHEST_PROTOCOL)
#!/bin/bash
#SBATCH --comment=773320000
#SBATCH --time=1-0
#SBATCH --mem-per-cpu=4000
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=1
#SBATCH --nodes=1
#SBATCH --array=0-49
#SBATCH --output=./out/%A_%a_output.txt
#SBATCH --error=./out/%A_%a_error_output.txt
#SBATCH --job-name=03051226
#SBATCH --mail-type=ALL
#SBATCH --mail-user=hans.roelofsen@wur.nl
# activate conda environment
source /home/WUR/roelo008/miniconda3/etc/profile.d/conda.sh
conda activate rasters
echo $SLURM_ARRAY_TASK_ID
# Run
cd /home/WUR/roelo008/projs/doren_2019
python run_species.py $SLURM_ARRAY_TASK_ID 50
\ No newline at end of file
from utils import doren_classes as dc
doren = dc.Doren(header_src=r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\a_brondata\EVA\EVA_Doren_header.csv',
sp_src=r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\a_brondata\EVA\EVA_Doren_species.csv')
doren.initiate()
doren.apply_requirements('req1', 'req2', 'req3', 'req4', 'req8', 'req9', 'req10',
aoi_src=r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\geodata\eva_aoi_fin_3035.shp',
dem_src=r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\a_brondata\covariables\DEM\DTM_3035.tif')
doren.add_posch(posch_src_dir=r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\a_brondata\POSCH_dep\v2')
doren.add_covar(covar_dir=r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\a_brondata\covariables\soil\b_processed',
covar_src='WRBLEV1_laea.tif', covar_name='soil_type', nominal=True)
doren.add_covar(covar_dir=r'c:\Users\roelo008\OneDrive - WageningenUR\b_geodata\eur_cntrs_3035',
covar_src='esri_eur_cntrs_3035.shp', covar_name='country', raster=False, column='CNTRYNAME',
keep_all=True)
doren.add_yearly_covar(covar_dir=r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\a_brondata\covariables\EObs\2_compiled',
covar_src_basename="EObs_v200e_rr_5yrmean", covar_name='5_yearly_precip')
doren.add_yearly_covar(covar_dir=r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\a_brondata\covariables\EObs\2_compiled',
covar_src_basename="EObs_v200e_tg_5yrmean", covar_name='5_yearly_temp')
doren.select_plts_w_species(species_name='Empetrum nigrum')
doren.filter_by_buffer_around_positive_plots(buffer_size=100000)
doren.write_stuff('species_PG')
doren.write_stuff('NDep_PG', covars=True)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Empetrum nigrum", buffer_size=10000)
doren.write_stuff('NDep_PG', covars=True)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Empetrum nigrum", buffer_size=25000)
doren.write_stuff('NDep_PG')
doren.write_stuff('NDep_PG', covars=True)
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Empetrum nigrum", buffer_size=50000)
doren.write_stuff('NDep_PG')
doren.write_stuff('NDep_PG', covars=True)
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Empetrum nigrum", buffer_size=100000)
doren.write_stuff('NDep_PG')
doren.write_stuff('NDep_PG', covars=True)
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Calluna vulgaris", buffer_size=5000)
doren.write_stuff('species_PG')
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Calluna vulgaris", buffer_size=10000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Calluna vulgaris", buffer_size=25000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Calluna vulgaris", buffer_size=50000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Calluna vulgaris", buffer_size=100000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Anemone nemorosa", buffer_size=5000)
doren.write_stuff('species_PG')
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Anemone nemorosa", buffer_size=10000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Anemone nemorosa", buffer_size=25000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Anemone nemorosa", buffer_size=50000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Anemone nemorosa", buffer_size=100000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Briza media", buffer_size=5000)
doren.write_stuff('species_PG')
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Briza media", buffer_size=10000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Briza media", buffer_size=25000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Briza media", buffer_size=50000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Briza media", buffer_size=100000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Vaccinium oxycoccos", buffer_size=5000)
doren.write_stuff('species_PG')
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Vaccinium oxycoccos", buffer_size=10000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Vaccinium oxycoccos", buffer_size=25000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Vaccinium oxycoccos", buffer_size=50000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Vaccinium oxycoccos", buffer_size=100000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Drosera rotundifolia", buffer_size=5000)
doren.write_stuff('species_PG')
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Drosera rotundifolia", buffer_size=10000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Drosera rotundifolia", buffer_size=25000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Drosera rotundifolia", buffer_size=50000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Drosera rotundifolia", buffer_size=100000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Puccinellia fasciculata", buffer_size=5000)
doren.write_stuff('species_PG')
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Puccinellia fasciculata", buffer_size=10000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Puccinellia fasciculata", buffer_size=25000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Puccinellia fasciculata", buffer_size=50000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Puccinellia fasciculata", buffer_size=100000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Centaurium pulchellum", buffer_size=5000)
doren.write_stuff('species_PG')
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Centaurium pulchellum", buffer_size=10000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Centaurium pulchellum", buffer_size=25000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Centaurium pulchellum", buffer_size=50000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Centaurium pulchellum", buffer_size=100000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Calamagrostis epigejos", buffer_size=5000)
doren.write_stuff('species_PG')
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Calamagrostis epigejos", buffer_size=10000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Calamagrostis epigejos", buffer_size=25000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Calamagrostis epigejos", buffer_size=50000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Calamagrostis epigejos", buffer_size=100000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Gentiana cruciata", buffer_size=5000)
doren.write_stuff('species_PG')
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Gentiana cruciata", buffer_size=10000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Gentiana cruciata", buffer_size=25000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Gentiana cruciata", buffer_size=50000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.filter_by_buffer(species_name="Gentiana cruciata", buffer_size=100000)
doren.write_stuff('NDep_PG')
doren.write_stuff("buffer_png")
doren.write_stuff('species_list')
doren.write_stuff('report')
"""
Script for taking a pre-existing DOREN pkl object and processing a subset of its requested species
Hans Roelofsen, 03 June 2020
"""
import numpy as np
import pickle
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('batch_nr', help='batch number')
parser.add_argument('n_batches', help='number of batches')
args = parser.parse_args()
batch_nr = int(args.batch_nr)
n_batches = int(args.n_batches)
# Recover the Pickled doren object
pkl_src = './pkl_src/doren_20200603.pkl'
with open(pkl_src, 'rb') as handle:
doren = pickle.load(handle)
doren.base_out_dir = r'./c_out'
# Determine subset of requested species to proces
sel = np.array_split(np.arange(0, len(doren.requested_species)), n_batches)[batch_nr].tolist()
print('Commencing processing of {0} species...'.format(len(sel)))
# Process subset of requested species
for i, j in enumerate(sel, start=1):
sp = doren.requested_species[j]
try:
print(' doing species {0} of {1}: {2}'.format(i, len(sel), sp))
doren.select_plts_w_species(species_name=sp)
doren.filter_by_buffer_around_positive_plots(buffer_size=100000)
doren.write_stuff('species_PG')
doren.write_stuff('NDep_PG', covars=True)
doren.write_stuff('NDep_PG')
except OSError:
continue
......@@ -50,6 +50,7 @@ class Doren:
self.verbose = verbose # boolean, report on progress?
self.report = '' # tracking everything in a report
# base directory for all output
self.requested_species = None #
self.base_out_dir = r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\z_scratch'
# source to shapefile with background images
self.background_shp = r'c:\Users\roelo008\OneDrive - WageningenUR\a_projects\DOREN\geodata\europe_3035.shp'
......@@ -102,6 +103,18 @@ class Doren:
self.report += 'Starting @ {0} with {1} EVA headers containing {2} unique ' \
'species.\n\n'.format(self.timestamp_full, self.status['n_plots'], self.status['n_species'])
def get_requested_species(self, src, **kwargs):
"""
read list of species requested for processing
:param src: file path to source document
:param kwargs:
:return:
"""
# assumed to be CSV sheet for now
src = pd.read_csv(src)
self.requested_species = list(set(src.wteneschappelijke_soortnaam))
def apply_requirements(self, *reqs, **kwargs):
"""
Filter down EVA headers by applying one or more requirements
......@@ -350,7 +363,7 @@ class Doren:
nearby_plots = gp.sjoin(left_df=self.eva.loc[self.negative_plots], right_df=buff_gdf, op='within',
how='inner').index
msg = 'Found {0} plots with {1} and {2} plots without inside {3}m radius buffer\n'\
msg = ' found {0} plots with {1} and {2} plots without inside {3}m radius buffer\n'\
.format(len(self.positive_plots), self.sel_species, len(nearby_plots), buffer_size)
self.report += msg
if self.verbose:
......@@ -440,7 +453,7 @@ class Doren:
# Filename = <self.buffer_species>.txt
sp_sel = pd.DataFrame(data={'plot_id': list(self.positive_plots),
'species_nr': self.species2nr[self.sel_species]}).sort_values(by='plot_id')
sp_sel.to_csv(os.path.join(pg_dir, pg_out_name), sep=',', index=False, header=False)
sp_sel.to_csv(os.path.join(pg_dir, pg_out_name), sep=' ', index=False, header=False)
self.report += 'Written FORTRAN input to files: {0}\n\n'.format(os.path.join(pg_dir, pg_out_name))
......@@ -468,7 +481,7 @@ class Doren:
else:
out_cols = ['totN_mol_ha']
self.eva.loc[self.positive_plots.union(self.nearby_plots), out_cols].sort_index(axis=0, ascending=True)\
.to_csv(os.path.join(pg_dir, '{}.txt'.format(pg_out_name)), sep='\t', index=True,
.to_csv(os.path.join(pg_dir, '{}.txt'.format(pg_out_name)), sep=',' if covars else ' ', index=True,
header=True if covars else False)
self.report += 'Written FORTRAN input to files: {0}\n\n'.format(os.path.join(pg_dir, pg_out_name))
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment