setup improvements

kubernetes/Dockerfile

@@ -118,9 +118,9 @@ COPY scripts /scripts
 COPY conda /conda
 
 # Set different location for R packages
-COPY scripts/Renviron /root/.Renviron
+# COPY scripts/Renviron /root/.Renviron
 
 # Extend bashrc
-RUN echo "export R_LIBS=/unlock/infrastructure/R/library/" >> ~/.bashrc
-RUN echo "export R_LIBS_SITE=/unlock/infrastructure/R/library/" >> ~/.bashrc
-RUN echo "export R_LIBS_USER=/unlock/infrastructure/R/library/root@3b440897a84b" >> ~/.bashrc
\ No newline at end of file
+RUN echo "export R_LIBS=/unlock/infrastructure/R/library" >> ~/.bashrc
+RUN echo "export R_LIBS_SITE=/unlock/infrastructure/R/library" >> ~/.bashrc
+RUN echo "export R_LIBS_USER=/unlock/infrastructure/R/library" >> ~/.bashrc
\ No newline at end of file

kubernetes/run.sh

@@ -56,7 +56,7 @@ python3 /scripts/metadata.py -key cwl -value $cwl -unit running -add -file $yaml
 [ $? -eq 0 ] && echo "Metadata added" || exit 1
 
 # Download latest yaml file
-java -jar /unlock/infrastructure/binaries/irods/IRODSTransfer.jar -pull --files $yaml --local $yaml_dir --force
+java -jar /unlock/infrastructure/binaries/irods/IRODSTransfer.jar -pull --files $yaml --local $yaml_dir
 [ $? -eq 0 ] && echo "downloaded $yaml" || exit 1
 
 # Obtain destination path
@@ -67,27 +67,18 @@ grep "[0-9]\+_irods:" $yaml | awk '{print $2}' | sort | uniq | xargs dirname | x
 # Download run files from iRODS
 grep "[0-9]\+_irods:" $yaml | awk '{print $2}' | sort | uniq | awk '{print "java -jar /unlock/infrastructure/binaries/irods/IRODSTransfer.jar --pull --local "$1" --files "$1" --force"}' | sh
 
-# Start a conda environment when needed
-conda_env=`grep conda $yaml | awk -F": " '{print $2}'`
-if ! [[ -z "$conda_env" ]]; then
-  source /root/miniconda/bin/activate && conda activate $conda_env
-fi
-
 # Run workflow with or without provenance
 if $prov; then
-    cwltool --tmpdir-prefix /unlock/rancher/unlockcwl --no-container --preserve-entire-environment --outdir $destination_path --provenance $destination_path/PROVENANCE $cwl $yaml
+    cwltool --cachedir CACHE --tmpdir-prefix /unlock/rancher/unlockcwl --no-container --preserve-entire-environment --outdir $destination_path --provenance $destination_path/PROVENANCE $cwl $yaml
 else
-    cwltool --tmpdir-prefix /unlock/rancher/unlockcwl --no-container --preserve-entire-environment --outdir $destination_path $cwl $yaml
+    cwltool --cachedir CACHE --tmpdir-prefix /unlock/rancher/unlockcwl --no-container --preserve-entire-environment --outdir $destination_path $cwl $yaml
 fi
 
 # Check exit status of cwl
 exit_status=$?
 
-# Start a conda environment when needed
-conda_env=`grep conda $yaml | awk -F": " '{print $2}'`
-if ! [[ -z "$conda_env" ]]; then
-  source /root/miniconda/bin/activate && conda deactivate
-fi
+# Make sure conda is deactivated afterwards in addition cwl scripts should do this on their own
+source /root/miniconda/bin/activate && conda deactivate
 
 # When success, update avu on yaml file and upload results
 if [ $exit_status -eq 0 ]; then

kubernetes/scripts/setup.sh

@@ -3,34 +3,32 @@
 # Ensures that R can use the library folder
 mkdir --parents /unlock/infrastructure/R/library
 
+################################################################
+# Conda environments
+################################################################
+
 # Update conda
 conda update -y -n base -c defaults conda
 
 ## Conda environment for picrust
 conda update -y -n base conda
 
-# picrust
-source /root/miniconda/bin/activate && \ 
-    cd /unlock/infrastructure/binaries/picrust2-2.4.1/ && \ 
-    conda env create -f /conda/picrust2_env.yml -p /unlock/infrastructure/conda/picrust2 && \ 
-    conda activate /unlock/infrastructure/conda/picrust2 && \ 
-    pip install --editable . && \ 
-    # Needed?
-    conda env update -p /unlock/infrastructure/conda/picrust2 --file /conda/picrust2_env.yml && \
-    conda deactivate
+# New attempt for picrust
+source /root/miniconda/bin/activate
+mamba create --yes --prefix /unlock/infrastructure/conda/picrust2 -c bioconda -c conda-forge picrust2=2.4.2
 
 # Krona
-mamba create --yes --prefix /unlock/infrastructure/conda/krona && \
-    mamba install --yes -c bioconda krona && \
-    ktUpdateTaxonomy.sh 
+mamba create --yes --prefix /unlock/infrastructure/conda/krona
+mamba install --yes -c bioconda krona
+ktUpdateTaxonomy.sh 
 
 # Medaka
-bash
 source /root/miniconda/bin/activate 
 mamba create --yes --prefix /unlock/infrastructure/conda/medaka -c bioconda -c conda-forge medaka=1.5.0
-# conda activate /unlock/infrastructure/conda/medaka
-# mamba install --yes -c bioconda -c conda-forge medaka=1.0.3
 
+################################################################
+# Pip environment
+################################################################
 
 # Python virtual environment
 python3 -m venv /unlock/infrastructure/venv
@@ -39,26 +37,26 @@ source /unlock/infrastructure/venv/bin/activate
 # Upgrade because we can
 /unlock/infrastructure/venv/bin/python3 -m pip install --upgrade pip
 
-## HUMAnN meta-omics
-python3 -m pip install humann --no-binary :all: && \ 
-	python3 -m pip install metaphlan --no-binary :all: && \
-	humann_config --update database_folders nucleotide /unlock/references/databases/HUMAnN/chocophlan && \
-	humann_config --update database_folders utility_mapping /unlock/references/databases/HUMAnN/utility_mapping && \
-	humann_config --update database_folders protein /unlock/references/databases/HUMAnN/uniref
+## HUMAnN 3 meta-omics only available via pip
+python3 -m pip install humann --no-binary :all:
+python3 -m pip install metaphlan --no-binary :all:
+humann_config --update database_folders nucleotide /unlock/references/databases/HUMAnN/chocophlan
+humann_config --update database_folders utility_mapping /unlock/references/databases/HUMAnN/utility_mapping
+humann_config --update database_folders protein /unlock/references/databases/HUMAnN/uniref
 
 ## Pip dependencies
-python3 -m pip install wheel biom-format pysam numpy matplotlib checkm-genome gtdbtk
+python3 -m pip install wheel biom-format pysam numpy matplotlib checkm-genome gtdbtk rdflib python-irodsclient html5lib
 
-python3 -m pip install cwltool==3.1.20211004060744 && \
-    cwltool --version && python3 -m pip install html5lib
+python3 -m pip install cwltool==3.1.20211004060744
+cwltool --version && python3 -m pip install html5lib
 
 ## Pip quast rc1 for 3.9 support
-wget https://github.com/ablab/quast/releases/download/quast_5.1.0rc1/quast-5.1.0rc1.tar.gz &&\
-    pip install quast-5.1.0rc1.tar.gz
+wget https://github.com/ablab/quast/releases/download/quast_5.1.0rc1/quast-5.1.0rc1.tar.gz
+pip install quast-5.1.0rc1.tar.gz
 
 ## CheckM
 checkm data setRoot /unlock/references/databases/CheckM/
 
 # R packages
 R -e "install.packages('docopt',dependencies=TRUE, repos='https://packagemanager.rstudio.com/all/__linux__/bionic/latest')"
-Rscript /scripts/install2.R -n 10 -s -l /unlock/infrastructure/R/library -r https://packagemanager.rstudio.com/all/__linux__/bionic/latest data.table futile.logger ggplot2 optparse plyr readr reshape2 scales viridis yaml parallel
\ No newline at end of file
+Rscript /scripts/install2.R -n 10 -s -l /unlock/infrastructure/R/library -r https://packagemanager.rstudio.com/all/__linux__/bionic/latest data.table futile.logger ggplot2 optparse plyr readr reshape2 scales viridis yaml parallel castor
\ No newline at end of file