Commit 216a8142 authored by Jasper Koehorst's avatar Jasper Koehorst
Browse files

folders created for sub groups of kubernetes docker for exeuction and docker for notebooks

parent c365e758
version: "3"
services:
irods:
container_name: irods
image: docker-registry.wur.nl/unlock/docker:irods
# command: start.sh jupyter lab
# ports:
# - "8888:8888"
# volumes:
# - unlock-data:/home/jovyan
rstudio:
container_name: rstudio
image: docker-registry.wur.nl/unlock/docker:rancher
ports:
- "8787:8787"
volumes:
- unlock-data:/home/rstudio
environment:
- PASSWORD=57Grv3xFc2E96nsehAJ4
- ROOT=TRUE
volumes:
unlock-data:
driver: local
driver_opts:
o: bind
type: none
device: ~/unlock-data
\ No newline at end of file
## auto-generated configuration file - DO NOT EDIT ##
/LIBS/GUID = "6ddf2e83-5943-4d67-acfd-ab83d2731cb6"
/config/default = "false"
/repository/user/ad/public/apps/file/volumes/flatAd = "."
/repository/user/ad/public/apps/refseq/volumes/refseqAd = "."
/repository/user/ad/public/apps/sra/volumes/sraAd = "."
/repository/user/ad/public/apps/sraPileup/volumes/ad = "."
/repository/user/ad/public/apps/sraRealign/volumes/ad = "."
/repository/user/ad/public/apps/wgs/volumes/wgsAd = "."
/repository/user/ad/public/root = "."
/repository/user/default-path = "/root/ncbi"
apiVersion: v1
kind: ServiceAccount
metadata:
name: gitlab
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: gitlab-admin
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: gitlab
namespace: kube-system
#!/bin/bash
usage() { echo "Usage: $0 [-c <cwl file path>] [-y <yaml file path>] [-p <true|false>]" 1>&2; exit 1; }
while getopts ":c:y:p:" o; do
case "${o}" in
c)
cwl=${OPTARG}
;;
y)
yaml=${OPTARG}
;;
p)
prov=${OPTARG}
((prov == true || prov == false)) || usage
;;
*)
usage
;;
esac
done
shift $((OPTIND-1))
if [ -z "${cwl}" ] || [ -z "${yaml}" ] || [ -z "${prov}" ]; then
usage
fi
echo "c = ${cwl}"
echo "y = ${yaml}"
echo "p = ${prov}"
echo $@
#######################################
yaml_filepath=$yaml
# Obtain parent directory
yaml_dir="$(dirname "$yaml")"/
# Obtain the yaml name
yaml_file_name="$(basename "$yaml")"
# Download the yaml file $1=yaml full path
mkdir -p $yaml_dir
# Delete yaml file if already exists
rm $yaml
# Updating metadata by removing all cwl and cwl path status
python3 /scripts/metadata.py -key cwl -value $cwl -remove -file $yaml
# Adding the running status
python3 /scripts/metadata.py -key cwl -value $cwl -unit running -add -file $yaml
# Download latest yaml file
java -jar /unlock/infrastructure/binaries/irods/IRODSTransfer.jar -pull --files $yaml --local $yaml_dir --force
# Obtain destination path
destination_path=`cat $yaml | grep destination | awk -F": " '{print $2}'`
# Create parent folders to store the files in
grep "[0-9]\+_irods:" $yaml | awk '{print $2}' | sort | uniq | xargs dirname | xargs mkdir -p
# Download run files from iRODS
grep "[0-9]\+_irods:" $yaml | awk '{print $2}' | sort | uniq | awk '{print "java -jar /unlock/infrastructure/binaries/irods/IRODSTransfer.jar --pull --local "$1" --files "$1" --force"}' | sh
# Start a conda environment when needed
conda_env=`grep conda $yaml | awk -F": " '{print $2}'`
if ! [[ -z "$conda" ]]; then
source /root/miniconda/bin/activate && conda activate $conda_env
fi
# Run workflow with or without provenance
if $prov; then
cwltool --tmpdir-prefix /unlock/rancher/unlockcwl --no-container --preserve-entire-environment --outdir $destination_path --provenance $destination_path/PROVENANCE $cwl $yaml
else
cwltool --tmpdir-prefix /unlock/rancher/unlockcwl --no-container --preserve-entire-environment --outdir $destination_path $cwl $yaml
fi
# Check exit status of cwl
exit_status=$?
# When success, update avu on yaml file and upload results
if [ $exit_status -eq 0 ]; then
# Remove provenance data file
if $prov; then
rm -r $destination_path/PROVENANCE/data/
fi
# Remove cache data
# rm -r $destination_path/CACHE_$yaml_file_name
# Upload the provenance $4=Final destination (Hash path)
java -jar /unlock/infrastructure/binaries/irods/IRODSTransfer.jar --push --files $destination_path --irods $destination_path
# Update metadata field
python3 /scripts/metadata.py -key cwl -value $cwl -remove -file $yaml
python3 /scripts/metadata.py -key cwl -value $cwl -unit finished -add -file $yaml
# Remove local destination results
rm -rf $destination_path
else
# When CWL execution fails update metadata
echo "CWL execution failed..."
python3 /scripts/metadata.py -key cwl -value $cwl -remove -file $yaml
python3 /scripts/metadata.py -key cwl -value $cwl -unit failed -add -file $yaml
exit 1
fi
# Remove downloaded files
echo "Removing irods files"
grep "[0-9]\+_irods:" $yaml | awk '{print "rm -f "$2}' | sh
exit 0
from irods.session import iRODSSession
import ssl
from irods.meta import iRODSMeta
import os, sys
import gzip
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("-file", help="Path to the file on the irods system of which the metadata needs to be modified", required=True)
args = parser.parse_args()
# iRODS authentication information
host = os.getenv('irodsHost')
port = os.getenv('irodsPort')
zone = os.getenv('irodsZone')
user = os.getenv('irodsUserName')
password = os.getenv('irodsPassword')
# SSL settings
context = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH, cafile=None, capath=None, cadata=None)
ssl_settings = {'irods_client_server_negotiation': 'request_server_negotiation',
'irods_client_server_policy': 'CS_NEG_REQUIRE',
'irods_encryption_algorithm': 'AES-256-CBC',
'irods_encryption_key_size': 32,
'irods_encryption_num_hash_rounds': 16,
'irods_encryption_salt_size': 8,
'ssl_context': context}
# Authentication
with iRODSSession(
host = host,
port = port,
user = user,
password = password,
zone = zone,
**ssl_settings) as session:
with gzip.open(args.file,'r') as fin:
count = 0
reads = 0
bases = 0
length = 0
for index, line in enumerate(fin):
count = count + 1
if count == 2:
line = line.decode("ascii")
reads = reads + 1
bases = bases + len(line.strip())
if len(line.strip()) > length:
length = len(line.strip())
if count == 4:
count = 0
directory = os.path.abspath(os.path.dirname(__file__))
command = "python3 " + directory + "/metadata.py -add -file " + args.file + " -key reads -value " + str(reads)
os.system(command)
command = "python3 " + directory + "/metadata.py -add -file " + args.file + " -key bases -value " + str(bases)
os.system(command)
command = "python3 " + directory + "/metadata.py -add -file " + args.file + " -key length -value " + str(length)
os.system(command)
\ No newline at end of file
version: "3"
services:
jupyter:
container_name: jupyter
image: docker-registry.wur.nl/unlock/docker:jupyter
command: start.sh jupyter lab
ports:
- "8888:8888"
volumes:
- unlock-data:/home/jovyan
rstudio:
container_name: rstudio
image: docker-registry.wur.nl/unlock/docker:rstudio
ports:
- "8787:8787"
volumes:
- unlock-data:/home/rstudio
environment:
- PASSWORD=57Grv3xFc2E96nsehAJ4
- ROOT=TRUE
graphdb:
container_name: graphdb
image: docker-registry.wur.nl/unlock/docker:graphdb
command: /graphdb-free/bin/graphdb -Dgraphdb.home=/unlock-data/graphdb
ports:
- "7200:7200"
volumes:
- unlock-data:/unlock-data
volumes:
unlock-data:
driver: local
driver_opts:
o: bind
type: none
device: ~/unlock-data
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment