Commit 35748084 authored by roelo008's avatar roelo008

adding ESD overall mananagment system

parent c86b4e1a
......@@ -43,6 +43,8 @@ import rasterio as rio
import numpy as np
from z_utils import esd
option = 2
"""Lees bronbestanden als Rasterio object en als numpy array. Definieer ook NoData"""
sl = rio.open(r'c:\apps\temp_geodata\lceu_huidig2020\sloten_top10\Sloten_2019_10m_255.tif')
sl_arr = sl.read(1)
......@@ -101,62 +103,56 @@ pc_cats_top10 = [0, 3, 3, 0, 0, 0, 0, 0, 3, 0, 0, 3, 4, 4, 0, 0, 0, 0, 0, 0, 0,
top10_2_pc = dict(zip(top10_cats, pc_cats_top10))
top10_pc_arr = esd.vec_translate(top10_arr, top10_2_pc)
'''==OPTIE1=== Define where each array pops up in the output'''
'''==OPTIE1=== Hierarchische classificatie'''
if option == 1:
# top10 where everyting else is NoData and Top10 self is not NoData
where_top10 = (bt_pc_arr == 0) & (anb_arr == 0) & (sl_arr == sl_nodata) & (top10_pc_arr != 0)
where_bt = (anb_arr == 0) & (sl_arr == sl_nodata) & (bt_pc_arr != 0) # bt where sloten AND anb is NoData
where_anb = (sl_arr == sl_nodata) & (anb_arr != 0) # anb where sloten is NoData
where_sl = sl_arr != sl_nodata # sloten where sloten is not NoData
out_arr = np.zeros_like(sl_arr).astype(np.uint8) # zeros array
out_arr[where_top10] = top10_pc_arr[where_top10] # put in top10 values where needed
out_arr[where_bt] = bt_pc_arr[where_bt] # put in bt values where needed
out_arr[where_anb] = anb_arr[where_anb] # put in anb values where needed
out_arr[where_sl] = sl_arr[where_sl] # put in sloten where needed
source_arr = np.zeros_like(sl_arr)
source_arr[where_sl] = 10 # alle cellen waar de waarde van sloten komt
source_arr[where_anb] = 20 # idem voor ANB
source_arr[where_bt] = 30 # idem voor BeheerType
source_arr[where_top10] = 40 # idem voor Top10
'''SAMPLE DATA
sloten = np.array(np.random.randint(low=0, high=4, size=49)).reshape(7, 7)
anb = np.array(np.random.randint(low=0, high=4, size=49)).reshape(7, 7)
bt = np.array(np.random.randint(low=0, high=4, size=49)).reshape(7, 7)
top10 = np.array(np.random.randint(low=0, high=4, size=49)).reshape(7, 7)
where_none = (bt == 0) & (sloten == 0) & (anb == 0) & (top10 == 0 )
where_top10 = (bt == 0) & (sloten == 0) & (anb == 0) & (top10 != 0)
where_bt = (anb == 0) & (sloten == 0) & (bt != 0)
where_anb = (sloten == 0) & (anb != 0)
where_sloten = (sloten != 0)
zeros = np.zeros_like(bt)
zeros[where_sloten] = 10
zeros[where_anb] = 20
zeros[where_bt] = 30
zeros[where_top10] = 40
zeros[where_none] = 0
'''
"""Write to file"""
out_dir = r'c:\apps\temp_geodata\lceu_huidig2020\out_pestcontrol_20200324'
prof = bt.profile
prof.update(dtype=rio.uint8)
with rio.open(os.path.join(out_dir, 'pest_control_land_use_20200324_v2.tif'), 'w', **prof) as dst:
dst.write(out_arr.astype(rio.uint8), 1)
with rio.open(os.path.join(out_dir, 'pest_control_land_use_20200324_sources.tif'),'w', **prof) as dst:
dst.write(source_arr.astype(rio.uint8), 1)
where_top10 = (bt_pc_arr == 0) & (anb_arr == 0) & (sl_arr == sl_nodata) & (top10_pc_arr != 0)
where_bt = (anb_arr == 0) & (sl_arr == sl_nodata) & (bt_pc_arr != 0) # bt where sloten AND anb is NoData
where_anb = (sl_arr == sl_nodata) & (anb_arr != 0) # anb where sloten is NoData
where_sl = sl_arr != sl_nodata # sloten where sloten is not NoData
out_arr = np.zeros_like(sl_arr).astype(np.uint8) # zeros array
out_arr[where_top10] = top10_pc_arr[where_top10] # put in top10 values where needed
out_arr[where_bt] = bt_pc_arr[where_bt] # put in bt values where needed
out_arr[where_anb] = anb_arr[where_anb] # put in anb values where needed
out_arr[where_sl] = sl_arr[where_sl] # put in sloten where needed
source_arr = np.zeros_like(sl_arr)
source_arr[where_sl] = 10 # alle cellen waar de waarde van sloten komt
source_arr[where_anb] = 20 # idem voor ANB
source_arr[where_bt] = 30 # idem voor BeheerType
source_arr[where_top10] = 40 # idem voor Top10
"""Write to file"""
out_dir = r'c:\apps\temp_geodata\lceu_huidig2020\out_pestcontrol_20200324'
prof = bt.profile
prof.update(dtype=rio.uint8)
with rio.open(os.path.join(out_dir, 'pest_control_land_use_20200324_v2.tif'), 'w', **prof) as dst:
dst.write(out_arr.astype(rio.uint8), 1)
with rio.open(os.path.join(out_dir, 'pest_control_land_use_20200324_sources.tif'),'w', **prof) as dst:
dst.write(source_arr.astype(rio.uint8), 1)
'''==OPTIE1=== ANY classificatie'''
if option == 2:
meerj_brij = np.where((bt_pc_arr == 2) | (anb_arr == 2) | (sl_arr == 2) | (top10_pc_arr == 2), 1, 0)
meerj_barm = np.where((bt_pc_arr == 3) | (anb_arr == 3) | (sl_arr == 3) | (top10_pc_arr == 3), 1, 0)
bosjes = np.where((bt_pc_arr == 4) | (anb_arr == 4) | (sl_arr == 4) | (top10_pc_arr == 4), 1, 0)
aleenst_br = np.where((bt_pc_arr == 5) | (anb_arr == 5) | (sl_arr == 5) | (top10_pc_arr == 5), 1, 0)
out_dir = r'c:\apps\temp_geodata\lceu_huidig2020\out_pestcontrol_20200329'
prof = bt.profile
prof.update(dtype=rio.uint8)
with rio.open(os.path.join(out_dir, 'pest_control_meerjarig_bloemrijk_20200329.tif'), 'w', **prof) as dst:
dst.write(meerj_brij.astype(rio.uint8), 1)
with rio.open(os.path.join(out_dir, 'pest_control_meerjarig_bloemarm_20200329.tif'), 'w', **prof) as dst:
dst.write(meerj_barm.astype(rio.uint8), 1)
with rio.open(os.path.join(out_dir, 'pest_control_meerjarig_bosjes_20200329.tif'), 'w', **prof) as dst:
dst.write(bosjes.astype(rio.uint8), 1)
with rio.open(os.path.join(out_dir, 'pest_control_alleenstaande_bomenrij_20200329.tif'), 'w', **prof) as dst:
dst.write(aleenst_br.astype(rio.uint8), 1)
......@@ -4,6 +4,7 @@ import os
import rasterio as rio
import numpy as np
def myear_tab():
'''
For use in model 2J Luchtfiltering
......
"""
Several helper functions for the ESD manager program
Hans Roelofsen 15 June 2020
"""
import os
import rasterio as rio
import geopandas as gp
def get_raster_profile(src):
return rio.open(src).profile
def get_shp_profile(src):
shp = gp.read_file(src)
return {'geom_type': shp.geom_type,
'ext': os.path.splitext(src)[1]} # what else?
"""
User interface for ESD manager
"""
import argparse
from z_utils import esd_management as manager
parser = argparse.ArgumentParser()
parser.add_argument('scenario', help='scenario name (w/o spaces)')
parser.add_argument('of', default='full', required=True)
parser.add_argument('verbose', action='store_true')
parser.add_argument('models', type=list, action='store', dest='list', help='model codes space seperated', required=True)
args = parser.parse_args()
broker = manager.Broker(scenario=args.scenario, verbose=args.verbose, of=args.verbose, *args.models)
broker.compile_io()
broker.execute()
"""
Adaptor and broker classes for calling and retrieving ESD models
Hans Roelofsen, 11 juni 2020
"""
import os
import shutil
import datetime
from z_utils import esd_resources as resources
inputs = resources.inputs()
outputs = resources.outputs()
model_names = resources.esd_names()
scenarios = resources.scenarios()
model_wds = resources.model_wds()
class Broker:
"""
Interface between 1 user (front) and n model adaptors (back)
"""
def __init__(self, scenario, verbose, of, *models):
# Basic information for an ESD ensemble run
self.operator = os.environ.get('USERNAME')
self.timestamp_full = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
self.timestamp_brief = datetime.datetime.now().strftime("%Y%m%d%H%M")
self.verbose = verbose
self.of = of
# Scenario name
self.scenario = scenario
# Path to the scenario data repository, subdirectory of the source repository
self.src_repository = r'w:\EcoSysDiensten_WEnR\data_repo'
self.scn_repository = os.path.join(self.src_repository, self.scenario)
# List of ESD models requested for execution. Verify if the model code is recognized
self.models = [model for model in models if model in [x for x in dir(model_names) if not x.startswith('_') ]]
# Initiate adaptors for all requested models
self.adaptors = [Adaptor(model_code) for model_code in self.models]
def compile_io(self):
"""
Instruct adaptors to compile input and output paths for their models, using mixture of default and scenario
data. Also verify if all provided input data meet the specified requirements
:return:
"""
for a in self.adaptors:
a.get_input_paths(scenario_name=self.scenario)
a.verify_inputs()
a.specify_outputs(timestamp=self.timestamp_brief, scenario_name=self.scenario)
def execute(self):
"""
Instruct adaptors to execture their model
:return:
"""
for a in self.adaptors:
a.execute()
class Adaptor:
"""
Adaptor for a ESD model
"""
def __init__(self, model_code):
self.model_code = model_code
self.model_name = getattr(model_names, self.model_code)
self.wd = getattr(model_wds, self.model_code) # working directory
self.output_dir = None
self.input = getattr(inputs, self.model_code)
self.output = getattr(outputs, self.model_code)
self.scenario_params = None
def get_input_paths(self, scenario_name='default'):
"""
Compile all input for the model as a mixture between defaults and scenario
return: updated self.inputs
where parameter_name is **name** of input and value is full path to the file
"""
# Retrieve default and sceneario values for the model input paramters
default_data = scenarios.datas['default']
scenario_data = scenarios.datas[scenario_name]
# update model inputs, trying scenario data first, else defaults
for variable_name, obj in self.input.items():
try:
variable_value = scenario_data[variable_name] # try to get scenario value
except KeyError:
try:
variable_value = default_data[variable_name] # get default value if above fails
except KeyError:
raise Exception('No default value found for variable name {}'.format(variable_name)) # fail
setattr(obj, 'path', variable_value)
self.input[variable_name] = obj
self.scenario_params = scenario_name
def verify_inputs(self):
"""
Verify if all provided input data comply to specifications regarding pixel size, ncol, nrow, etc.
:return: Boolean
"""
for k, v in self.input:
if isinstance(v, resources.RasterProfile) or isinstance(v, resources.ShapeProfile):
v.verify_against_src(src=self.input[k].path)
def specify_outputs(self, timestamp, scenario_name='default'):
"""
Generate output names reflecting scenario etc
:return: updates self.outputs
"""
self.output_dir = os.path.join(self.wd, 'out_{0}'.format(timestamp))
if not os.path.isdir(self.output_dir):
os.mkdir(self.output_dir)
for variable_name, variable_value in self.output.items():
basename, ext = os.path.splitext(variable_value)
out_name = '{0}_{1}_{2}.{3}'.format(basename, scenario_name, timestamp, ext)
self.output[variable_name] = os.path.join(self.output_dir, out_name)
def compile_ini(selfs):
"""
Compile ini file with all settings for the model using self.inputs and self.outputs. Copy file to self.wd
:return: *.ini written to model workspace
"""
def transfer_input_data(self):
"""
Copy all parameter_values of self.inputs to self.wd
"""
for _, v in self.input.items():
location, fname = os.path.split(v)
shutil.copy(v, os.path.join(self.wd, fname))
def execute(self):
"""
execute the model using command line
c:/apps/model2J/2J.exe ini.ini
"""
print('..executing model {0} with parameters for scenario {1}'.format(self.model_name, self.scenario_params))
"""
Resources for the ESD model adaptors and broker
Hans Roelofsen, 15 June 2020
"""
from z_utils import esd_functions as esd
class inputs:
"""
Overview of input variables and parameters as required by each ESD model.
"""
def __init__(self):
self.AB = {'land_cover_scenario': RasterProfile(res=10, width=1000, height=1500, driver='GTiff')}
self.BJ = {
"land_cover_original": RasterProfile(res=10, width=1000, height=1500, driver='GTiff'),
"land_cover_scenario": RasterProfile(res=10, width=1000, height=1500, driver='GTiff'),
"rpm_2.5um_concentratie": RasterProfile(res=10, width=1000, height=1500, driver='GTiff'),
"cbs_inwoners": ShapeProfile(geom_type='ply', ext='.shp'),
"concentratie_norm": None,
"nl_area_onshore": None,
"m_year_tabel": None
}
class Outputs:
"""
Overview of output variables for ESD models
"""
def __init__(self):
self.BJ = {
"concentratie_blanco": '*.tif',
"concentratie_scenario": '*.tif',
"concentratie_scenario_norm": '*.tif',
"areaal_norm": '*.txt',
"bewoners_norm": '*.txt'
}
class ModelWorkingDirs:
"""
Working directories of EDS models
"""
def __init__(self):
self.BJ = r'c:/apps/modelBJ'
class ModelNames:
"""
Mapping ESD model codes to names
"""
def __init__(self):
self.BJ = 'Luchtzuivering'
self.BA = 'Bodemvruchtbaarheid'
self.BD = 'Verkoeling in de stad'
class Scenarios:
def __init__(self):
self.overview = ['default', 'BAU', 'HDBa', 'HDBb']
self.datas = {
"default": {"beheertypenkaart": None,
"LCEU": None,
"rpm_2.5um_concentratie": None,
"cbs_inwoners": None,
"concentratie_norm": None,
"nl_area_onshore": None,
"m_year_tabel": None,
"land_cover_original": 'LCEU.tif'},
"BAU": {'lceu': 'lceu_BAU.tif'},
"HDBa": {'lcea': 'lceu_HDBa.tif'}}
def compose_new(self, name, directory, **resources):
self.overview.append(name)
self.datas['name'] = resources
class RasterProfile:
"""
Class for holding all specifications of a geospatial raster file and method for checking provided rasters w. specs
"""
def __init__(self, res, width, height, driver):
self.path = None
self.res = res
self.width = width
self.height = height
self.driver = driver
self.verified = False
def verify_against_src(self, src):
"""
Check if a geospatial raster provided in src complies to self.res, self.width, self.height
:param src: path to a geospatial raster on file
:return: boolean
"""
src_prof = esd.get_raster_profile(src)
checks = {
"res_check": True if src_prof['affine'][0] == self.res else False,
"width_check": True if src_prof['width'] == self.width else False,
"height_check": True if src_prof['height'] == self.height else False,
"ext_check": True if src_prof['drive'] == self.driver else False}
if all([v for _, v in checks.items()]):
self.verified = True
else:
msg = 'Raster {0} failed check for {1}'.format(src, [k for k, v in checks.items() if not v])
raise Exception(msg)
class ShapeProfile:
"""
Class for holding all specifications of a shapefile and method for checking provided shps w. specs
"""
def __init__(self, geom_type, ext):
self.path = None
self.geom_type = geom_type
self.ext = ext
self.verified = False
def verify_against_src(self, src):
src = esd.get_shp_profile(src)
checks = {
'geom': True if src['geom'] == self.geom_type else False,
'ext': True if src['ext'] == self.ext else False}
if all([v for _, v in checks.items()]):
self.verified = True
else:
msg = 'Raster {0} failed check for {1}'.format(src, [k for k, v in checks.items() if not v])
raise Exception(msg)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment