Skip to content
Snippets Groups Projects
Commit e25e2ad7 authored by Bart's avatar Bart
Browse files

small fixes

parent 24f13880
Branches
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
notebooks/samsa2-prepare.ipynb
+ 14
− 69
View file @ e25e2ad7
%% Cell type:markdown id:seven-dairy tags: %% Cell type:code id:unique-creek tags:
# Retrieve SAMSA2 Output Files
%% Cell type:code id:lasting-asset tags:
``` python
# Dependencies
import os
import logging
from irods.models import DataObject, DataObjectMeta, Collection, CollectionMeta
from irods.column import Criterion
import getpass
from shutil import copyfileobj
import pathlib
```
%% Cell type:code id:private-proposal tags:
``` python ``` python
# Generic functions # Generic functions
LOGGING_FORMAT = '[%(asctime)-15s][%(levelname)-7s] %(message)s' LOGGING_FORMAT = '[%(asctime)-15s][%(levelname)-7s] %(message)s'
def get_logger(name): def get_logger(name):
logging.basicConfig(format=LOGGING_FORMAT) logging.basicConfig(format=LOGGING_FORMAT)
logger = logging.getLogger(name) logger = logging.getLogger(name)
logger.setLevel('DEBUG') logger.setLevel('DEBUG')
return logger return logger
logger = get_logger("jupyter") logger = get_logger("jupyter")
``` ```
%% Cell type:code id:informal-albert tags: %% Cell type:code id:shaped-perry tags:
``` python ``` python
# iRODS authentication # iRODS authentication
``` ```
%% Cell type:code id:pacific-peter tags: %% Cell type:code id:formed-parliament tags:
``` python ``` python
print("Username") print("Username")
username=input() username=input()
``` ```
%% Output %% Cell type:code id:turkish-probe tags:
Username
nijsse
%% Cell type:code id:minus-stuff tags:
``` python ``` python
print("Password") print("Password")
password = getpass.getpass() password = getpass.getpass()
``` ```
%% Output %% Cell type:code id:standing-shannon tags:
Password
············
%% Cell type:code id:functioning-climb tags:
``` python ``` python
from irods.session import iRODSSession from irods.session import iRODSSession
import ssl import ssl
# iRODS authentication information # iRODS authentication information
host = "unlock-icat.irods.surfsara.nl" host = "unlock-icat.irods.surfsara.nl"
port = "1247" port = "1247"
zone = "unlock" zone = "unlock"
context = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH, cafile=None, capath=None, cadata=None) context = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH, cafile=None, capath=None, cadata=None)
ssl_settings = {'irods_client_server_negotiation': 'request_server_negotiation', ssl_settings = {'irods_client_server_negotiation': 'request_server_negotiation',
'irods_client_server_policy': 'CS_NEG_REQUIRE', 'irods_client_server_policy': 'CS_NEG_REQUIRE',
'irods_encryption_algorithm': 'AES-256-CBC', 'irods_encryption_algorithm': 'AES-256-CBC',
'irods_encryption_key_size': 32, 'irods_encryption_key_size': 32,
'irods_encryption_num_hash_rounds': 16, 'irods_encryption_num_hash_rounds': 16,
'irods_encryption_salt_size': 8, 'irods_encryption_salt_size': 8,
'ssl_context': context} 'ssl_context': context}
session = iRODSSession(host = host, session = iRODSSession(host = host,
port = port, port = port,
user = username, user = username,
password = password, password = password,
zone = zone, zone = zone,
**ssl_settings) **ssl_settings)
``` ```
%% Cell type:code id:tested-detector tags: %% Cell type:code id:analyzed-wheat tags:
``` python ``` python
# Set the study # Set the study
project = "P_Deltares" project = "P_UNLOCK"
investigation = "I_vitens" investigation = "I_INVESTIGATION_TEST"
study = "S_ground_surface_water" study = "S_TEST_STUDY"
studyPath = "/unlock/projects/"+project+"/"+investigation+"/"+study studyPath = "/unlock/projects/"+project+"/"+investigation+"/"+study
``` ```
%% Cell type:code id:excessive-injury tags: %% Cell type:code id:overall-possibility tags:
``` python ``` python
# Obtain all SAMSA2 files for a given study # Obtain all SAMSA2 files for a given study
logger.info("Started collecting SAMSA2 output files") logger.info("Started collecting SAMSA2 output files")
types = ['function','organism'] types = ['function','organism']
for type in types: for type in types:
org_results = session.query(Collection, DataObject).filter( \ org_results = session.query(Collection, DataObject).filter( \
Criterion('like', DataObject.path,"%_"+type+".tsv")).filter( \ Criterion('like', DataObject.path,"%_"+type+".tsv")).filter( \
Criterion('like', Collection.name, studyPath + "/%")) Criterion('like', Collection.name, studyPath + "/%"))
# create output folders # create output folders
output_dir = "./data/"+project+"/"+investigation+"/"+study+"/samsa2_"+type+"/" output_dir = "./data/"+project+"/"+investigation+"/"+study+"/samsa2_"+type+"/"
pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True) pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True)
# Obtaining all files # Obtaining all files
index = 0 index = 0
for index, r in enumerate(org_results): for index, r in enumerate(org_results):
irods_filepath = r.get(Collection.name) + "/" +r.get(DataObject.name) irods_filepath = r.get(Collection.name) + "/" +r.get(DataObject.name)
obj = session.data_objects.get(irods_filepath) obj = session.data_objects.get(irods_filepath)
output_file = output_dir+"/"+r.get(DataObject.name) output_file = output_dir+"/"+r.get(DataObject.name)
with open(output_file, 'wb') as output, obj.open('r+') as input: with open(output_file, 'wb') as output, obj.open('r+') as input:
copyfileobj(input, output) copyfileobj(input, output)
logger.info("\nObtained " + str(index) + " SAMSA2 " + type + " files from study " + study + "\nFiles can be found here: " + output_dir) logger.info("\nObtained " + str(index) + " SAMSA2 " + type + " files from study " + study + "\nFiles can be found here: " + output_dir)
``` ```
%% Output %% Output
[2021-03-31 15:07:48,336][INFO ] Started collecting SAMSA2 output files [2021-03-31 15:07:48,336][INFO ] Started collecting SAMSA2 output files
[2021-03-31 15:08:22,388][INFO ] [2021-03-31 15:08:22,388][INFO ]
Obtained 27 SAMSA2 function files from study S_ground_surface_water Obtained 27 SAMSA2 function files from study S_ground_surface_water
Files can be found here: ./data/P_Deltares/I_vitens/S_ground_surface_water/samsa2_function/ Files can be found here: ./data/P_Deltares/I_vitens/S_ground_surface_water/samsa2_function/
[2021-03-31 15:08:45,119][INFO ] [2021-03-31 15:08:45,119][INFO ]
Obtained 27 SAMSA2 organism files from study S_ground_surface_water Obtained 27 SAMSA2 organism files from study S_ground_surface_water
Files can be found here: ./data/P_Deltares/I_vitens/S_ground_surface_water/samsa2_organism/ Files can be found here: ./data/P_Deltares/I_vitens/S_ground_surface_water/samsa2_organism/
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment