From 70ff524ba5f5ec30688baffb5c8798e797dae75c Mon Sep 17 00:00:00 2001 From: lcalisto <luis.calisto@hotmail.com> Date: Wed, 12 Mar 2025 11:28:34 +0000 Subject: [PATCH] updated concurrent tasks --- etl/etl.py | 6 +++--- etl/field_data/utils.py | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/etl/etl.py b/etl/etl.py index bc01c47..62c4c17 100755 --- a/etl/etl.py +++ b/etl/etl.py @@ -209,7 +209,7 @@ async def process_data(field_data_file_path='', soil_chem_data_file_path=None, s port (str, optional): Database port. Defaults to '5432'. sslmode (str, optional): Database sslmode. Defaults to 'require'. truncate (bool, optional): Option to truncate tables. Defaults to False. - max_concurrent_tasks (int, optional): Maximum number of concurrent tasks. Defaults to 10. + max_concurrent_tasks (int, optional): Maximum number of concurrent tasks. Defaults to 50. """ logging.debug('ETL started for field data') start_time = time.time() @@ -249,8 +249,8 @@ async def process_data(field_data_file_path='', soil_chem_data_file_path=None, s password=password, port=port, ssl=sslmode, - min_size=10, - max_size=50 + min_size=1, + max_size=max_concurrent_tasks ) # Truncate tables, delete all existing data from the tables diff --git a/etl/field_data/utils.py b/etl/field_data/utils.py index 24278d6..1f7e0c4 100644 --- a/etl/field_data/utils.py +++ b/etl/field_data/utils.py @@ -33,10 +33,14 @@ async def insert_individual(conn, name, email=None): """ try: result = await conn.fetchrow(sql, name, email) + if result is None: + logging.error('No result returned for individual: %s with email: %s', name, email) + return None inserted_individual_id = result['individual_id'] logging.debug('Individual: %s with id %s successfully inserted.', name, inserted_individual_id) return inserted_individual_id except asyncpg.PostgresError as err: + logging.error("An exception occurred trying to insert an Individual: %s", err) raise SystemExit("An exception occurred trying to insert an Individual. ", err) async def insert_address(conn, street_address, country_iso): -- GitLab