From 70ff524ba5f5ec30688baffb5c8798e797dae75c Mon Sep 17 00:00:00 2001
From: lcalisto <luis.calisto@hotmail.com>
Date: Wed, 12 Mar 2025 11:28:34 +0000
Subject: [PATCH] updated concurrent tasks

---
 etl/etl.py              | 6 +++---
 etl/field_data/utils.py | 4 ++++
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/etl/etl.py b/etl/etl.py
index bc01c47..62c4c17 100755
--- a/etl/etl.py
+++ b/etl/etl.py
@@ -209,7 +209,7 @@ async def process_data(field_data_file_path='', soil_chem_data_file_path=None, s
         port (str, optional): Database port. Defaults to '5432'.
         sslmode (str, optional): Database sslmode. Defaults to 'require'.
         truncate (bool, optional): Option to truncate tables. Defaults to False.
-        max_concurrent_tasks (int, optional): Maximum number of concurrent tasks. Defaults to 10.
+        max_concurrent_tasks (int, optional): Maximum number of concurrent tasks. Defaults to 50.
     """
     logging.debug('ETL started for field data')
     start_time = time.time()
@@ -249,8 +249,8 @@ async def process_data(field_data_file_path='', soil_chem_data_file_path=None, s
         password=password,
         port=port,
         ssl=sslmode,
-        min_size=10,
-        max_size=50
+        min_size=1,
+        max_size=max_concurrent_tasks
     )
 
     # Truncate tables, delete all existing data from the tables
diff --git a/etl/field_data/utils.py b/etl/field_data/utils.py
index 24278d6..1f7e0c4 100644
--- a/etl/field_data/utils.py
+++ b/etl/field_data/utils.py
@@ -33,10 +33,14 @@ async def insert_individual(conn, name, email=None):
     """
     try:
         result = await conn.fetchrow(sql, name, email)
+        if result is None:
+            logging.error('No result returned for individual: %s with email: %s', name, email)
+            return None
         inserted_individual_id = result['individual_id']
         logging.debug('Individual: %s with id %s successfully inserted.', name, inserted_individual_id)
         return inserted_individual_id
     except asyncpg.PostgresError as err:
+        logging.error("An exception occurred trying to insert an Individual: %s", err)
         raise SystemExit("An exception occurred trying to insert an Individual. ", err)
 
 async def insert_address(conn, street_address, country_iso):
-- 
GitLab