Skip to content
Snippets Groups Projects
Commit 128227fb authored by Jasper Koehorst's avatar Jasper Koehorst
Browse files

setup improvements

parent 48ff859d
Branches nanopore
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
kubernetes/Dockerfile
+ 4
− 4
View file @ 128227fb
......@@ -118,9 +118,9 @@ COPY scripts /scripts
COPY conda /conda
# Set different location for R packages
COPY scripts/Renviron /root/.Renviron
# COPY scripts/Renviron /root/.Renviron
# Extend bashrc
RUN echo "export R_LIBS=/unlock/infrastructure/R/library/" >> ~/.bashrc
RUN echo "export R_LIBS_SITE=/unlock/infrastructure/R/library/" >> ~/.bashrc
RUN echo "export R_LIBS_USER=/unlock/infrastructure/R/library/root@3b440897a84b" >> ~/.bashrc
\ No newline at end of file
RUN echo "export R_LIBS=/unlock/infrastructure/R/library" >> ~/.bashrc
RUN echo "export R_LIBS_SITE=/unlock/infrastructure/R/library" >> ~/.bashrc
RUN echo "export R_LIBS_USER=/unlock/infrastructure/R/library" >> ~/.bashrc
\ No newline at end of file
kubernetes/run.sh
+ 5
− 14
View file @ 128227fb
......@@ -56,7 +56,7 @@ python3 /scripts/metadata.py -key cwl -value $cwl -unit running -add -file $yaml
[ $? -eq 0 ] && echo "Metadata added" || exit 1
# Download latest yaml file
java -jar /unlock/infrastructure/binaries/irods/IRODSTransfer.jar -pull --files $yaml --local $yaml_dir --force
java -jar /unlock/infrastructure/binaries/irods/IRODSTransfer.jar -pull --files $yaml --local $yaml_dir
[ $? -eq 0 ] && echo "downloaded $yaml" || exit 1
# Obtain destination path
......@@ -67,27 +67,18 @@ grep "[0-9]\+_irods:" $yaml | awk '{print $2}' | sort | uniq | xargs dirname | x
# Download run files from iRODS
grep "[0-9]\+_irods:" $yaml | awk '{print $2}' | sort | uniq | awk '{print "java -jar /unlock/infrastructure/binaries/irods/IRODSTransfer.jar --pull --local "$1" --files "$1" --force"}' | sh
# Start a conda environment when needed
conda_env=`grep conda $yaml | awk -F": " '{print $2}'`
if ! [[ -z "$conda_env" ]]; then
source /root/miniconda/bin/activate && conda activate $conda_env
fi
# Run workflow with or without provenance
if $prov; then
cwltool --tmpdir-prefix /unlock/rancher/unlockcwl --no-container --preserve-entire-environment --outdir $destination_path --provenance $destination_path/PROVENANCE $cwl $yaml
cwltool --cachedir CACHE --tmpdir-prefix /unlock/rancher/unlockcwl --no-container --preserve-entire-environment --outdir $destination_path --provenance $destination_path/PROVENANCE $cwl $yaml
else
cwltool --tmpdir-prefix /unlock/rancher/unlockcwl --no-container --preserve-entire-environment --outdir $destination_path $cwl $yaml
cwltool --cachedir CACHE --tmpdir-prefix /unlock/rancher/unlockcwl --no-container --preserve-entire-environment --outdir $destination_path $cwl $yaml
fi
# Check exit status of cwl
exit_status=$?
# Start a conda environment when needed
conda_env=`grep conda $yaml | awk -F": " '{print $2}'`
if ! [[ -z "$conda_env" ]]; then
source /root/miniconda/bin/activate && conda deactivate
fi
# Make sure conda is deactivated afterwards in addition cwl scripts should do this on their own
source /root/miniconda/bin/activate && conda deactivate
# When success, update avu on yaml file and upload results
if [ $exit_status -eq 0 ]; then
......
kubernetes/scripts/setup.sh
+ 25
− 27
View file @ 128227fb
......@@ -3,34 +3,32 @@
# Ensures that R can use the library folder
mkdir --parents /unlock/infrastructure/R/library
################################################################
# Conda environments
################################################################
# Update conda
conda update -y -n base -c defaults conda
## Conda environment for picrust
conda update -y -n base conda
# picrust
source /root/miniconda/bin/activate && \
cd /unlock/infrastructure/binaries/picrust2-2.4.1/ && \
conda env create -f /conda/picrust2_env.yml -p /unlock/infrastructure/conda/picrust2 && \
conda activate /unlock/infrastructure/conda/picrust2 && \
pip install --editable . && \
# Needed?
conda env update -p /unlock/infrastructure/conda/picrust2 --file /conda/picrust2_env.yml && \
conda deactivate
# New attempt for picrust
source /root/miniconda/bin/activate
mamba create --yes --prefix /unlock/infrastructure/conda/picrust2 -c bioconda -c conda-forge picrust2=2.4.2
# Krona
mamba create --yes --prefix /unlock/infrastructure/conda/krona && \
mamba install --yes -c bioconda krona && \
ktUpdateTaxonomy.sh
mamba create --yes --prefix /unlock/infrastructure/conda/krona
mamba install --yes -c bioconda krona
ktUpdateTaxonomy.sh
# Medaka
bash
source /root/miniconda/bin/activate
mamba create --yes --prefix /unlock/infrastructure/conda/medaka -c bioconda -c conda-forge medaka=1.5.0
# conda activate /unlock/infrastructure/conda/medaka
# mamba install --yes -c bioconda -c conda-forge medaka=1.0.3
################################################################
# Pip environment
################################################################
# Python virtual environment
python3 -m venv /unlock/infrastructure/venv
......@@ -39,26 +37,26 @@ source /unlock/infrastructure/venv/bin/activate
# Upgrade because we can
/unlock/infrastructure/venv/bin/python3 -m pip install --upgrade pip
## HUMAnN meta-omics
python3 -m pip install humann --no-binary :all: && \
python3 -m pip install metaphlan --no-binary :all: && \
humann_config --update database_folders nucleotide /unlock/references/databases/HUMAnN/chocophlan && \
humann_config --update database_folders utility_mapping /unlock/references/databases/HUMAnN/utility_mapping && \
humann_config --update database_folders protein /unlock/references/databases/HUMAnN/uniref
## HUMAnN 3 meta-omics only available via pip
python3 -m pip install humann --no-binary :all:
python3 -m pip install metaphlan --no-binary :all:
humann_config --update database_folders nucleotide /unlock/references/databases/HUMAnN/chocophlan
humann_config --update database_folders utility_mapping /unlock/references/databases/HUMAnN/utility_mapping
humann_config --update database_folders protein /unlock/references/databases/HUMAnN/uniref
## Pip dependencies
python3 -m pip install wheel biom-format pysam numpy matplotlib checkm-genome gtdbtk
python3 -m pip install wheel biom-format pysam numpy matplotlib checkm-genome gtdbtk rdflib python-irodsclient html5lib
python3 -m pip install cwltool==3.1.20211004060744 && \
cwltool --version && python3 -m pip install html5lib
python3 -m pip install cwltool==3.1.20211004060744
cwltool --version && python3 -m pip install html5lib
## Pip quast rc1 for 3.9 support
wget https://github.com/ablab/quast/releases/download/quast_5.1.0rc1/quast-5.1.0rc1.tar.gz &&\
pip install quast-5.1.0rc1.tar.gz
wget https://github.com/ablab/quast/releases/download/quast_5.1.0rc1/quast-5.1.0rc1.tar.gz
pip install quast-5.1.0rc1.tar.gz
## CheckM
checkm data setRoot /unlock/references/databases/CheckM/
# R packages
R -e "install.packages('docopt',dependencies=TRUE, repos='https://packagemanager.rstudio.com/all/__linux__/bionic/latest')"
Rscript /scripts/install2.R -n 10 -s -l /unlock/infrastructure/R/library -r https://packagemanager.rstudio.com/all/__linux__/bionic/latest data.table futile.logger ggplot2 optparse plyr readr reshape2 scales viridis yaml parallel
\ No newline at end of file
Rscript /scripts/install2.R -n 10 -s -l /unlock/infrastructure/R/library -r https://packagemanager.rstudio.com/all/__linux__/bionic/latest data.table futile.logger ggplot2 optparse plyr readr reshape2 scales viridis yaml parallel castor
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment