Commit 128227fb authored by Jasper Koehorst's avatar Jasper Koehorst
Browse files

setup improvements

parent 48ff859d
......@@ -118,9 +118,9 @@ COPY scripts /scripts
COPY conda /conda
# Set different location for R packages
COPY scripts/Renviron /root/.Renviron
# COPY scripts/Renviron /root/.Renviron
# Extend bashrc
RUN echo "export R_LIBS=/unlock/infrastructure/R/library/" >> ~/.bashrc
RUN echo "export R_LIBS_SITE=/unlock/infrastructure/R/library/" >> ~/.bashrc
RUN echo "export R_LIBS_USER=/unlock/infrastructure/R/library/root@3b440897a84b" >> ~/.bashrc
\ No newline at end of file
RUN echo "export R_LIBS=/unlock/infrastructure/R/library" >> ~/.bashrc
RUN echo "export R_LIBS_SITE=/unlock/infrastructure/R/library" >> ~/.bashrc
RUN echo "export R_LIBS_USER=/unlock/infrastructure/R/library" >> ~/.bashrc
\ No newline at end of file
......@@ -56,7 +56,7 @@ python3 /scripts/metadata.py -key cwl -value $cwl -unit running -add -file $yaml
[ $? -eq 0 ] && echo "Metadata added" || exit 1
# Download latest yaml file
java -jar /unlock/infrastructure/binaries/irods/IRODSTransfer.jar -pull --files $yaml --local $yaml_dir --force
java -jar /unlock/infrastructure/binaries/irods/IRODSTransfer.jar -pull --files $yaml --local $yaml_dir
[ $? -eq 0 ] && echo "downloaded $yaml" || exit 1
# Obtain destination path
......@@ -67,27 +67,18 @@ grep "[0-9]\+_irods:" $yaml | awk '{print $2}' | sort | uniq | xargs dirname | x
# Download run files from iRODS
grep "[0-9]\+_irods:" $yaml | awk '{print $2}' | sort | uniq | awk '{print "java -jar /unlock/infrastructure/binaries/irods/IRODSTransfer.jar --pull --local "$1" --files "$1" --force"}' | sh
# Start a conda environment when needed
conda_env=`grep conda $yaml | awk -F": " '{print $2}'`
if ! [[ -z "$conda_env" ]]; then
source /root/miniconda/bin/activate && conda activate $conda_env
fi
# Run workflow with or without provenance
if $prov; then
cwltool --tmpdir-prefix /unlock/rancher/unlockcwl --no-container --preserve-entire-environment --outdir $destination_path --provenance $destination_path/PROVENANCE $cwl $yaml
cwltool --cachedir CACHE --tmpdir-prefix /unlock/rancher/unlockcwl --no-container --preserve-entire-environment --outdir $destination_path --provenance $destination_path/PROVENANCE $cwl $yaml
else
cwltool --tmpdir-prefix /unlock/rancher/unlockcwl --no-container --preserve-entire-environment --outdir $destination_path $cwl $yaml
cwltool --cachedir CACHE --tmpdir-prefix /unlock/rancher/unlockcwl --no-container --preserve-entire-environment --outdir $destination_path $cwl $yaml
fi
# Check exit status of cwl
exit_status=$?
# Start a conda environment when needed
conda_env=`grep conda $yaml | awk -F": " '{print $2}'`
if ! [[ -z "$conda_env" ]]; then
source /root/miniconda/bin/activate && conda deactivate
fi
# Make sure conda is deactivated afterwards in addition cwl scripts should do this on their own
source /root/miniconda/bin/activate && conda deactivate
# When success, update avu on yaml file and upload results
if [ $exit_status -eq 0 ]; then
......
......@@ -3,34 +3,32 @@
# Ensures that R can use the library folder
mkdir --parents /unlock/infrastructure/R/library
################################################################
# Conda environments
################################################################
# Update conda
conda update -y -n base -c defaults conda
## Conda environment for picrust
conda update -y -n base conda
# picrust
source /root/miniconda/bin/activate && \
cd /unlock/infrastructure/binaries/picrust2-2.4.1/ && \
conda env create -f /conda/picrust2_env.yml -p /unlock/infrastructure/conda/picrust2 && \
conda activate /unlock/infrastructure/conda/picrust2 && \
pip install --editable . && \
# Needed?
conda env update -p /unlock/infrastructure/conda/picrust2 --file /conda/picrust2_env.yml && \
conda deactivate
# New attempt for picrust
source /root/miniconda/bin/activate
mamba create --yes --prefix /unlock/infrastructure/conda/picrust2 -c bioconda -c conda-forge picrust2=2.4.2
# Krona
mamba create --yes --prefix /unlock/infrastructure/conda/krona && \
mamba install --yes -c bioconda krona && \
ktUpdateTaxonomy.sh
mamba create --yes --prefix /unlock/infrastructure/conda/krona
mamba install --yes -c bioconda krona
ktUpdateTaxonomy.sh
# Medaka
bash
source /root/miniconda/bin/activate
mamba create --yes --prefix /unlock/infrastructure/conda/medaka -c bioconda -c conda-forge medaka=1.5.0
# conda activate /unlock/infrastructure/conda/medaka
# mamba install --yes -c bioconda -c conda-forge medaka=1.0.3
################################################################
# Pip environment
################################################################
# Python virtual environment
python3 -m venv /unlock/infrastructure/venv
......@@ -39,26 +37,26 @@ source /unlock/infrastructure/venv/bin/activate
# Upgrade because we can
/unlock/infrastructure/venv/bin/python3 -m pip install --upgrade pip
## HUMAnN meta-omics
python3 -m pip install humann --no-binary :all: && \
python3 -m pip install metaphlan --no-binary :all: && \
humann_config --update database_folders nucleotide /unlock/references/databases/HUMAnN/chocophlan && \
humann_config --update database_folders utility_mapping /unlock/references/databases/HUMAnN/utility_mapping && \
humann_config --update database_folders protein /unlock/references/databases/HUMAnN/uniref
## HUMAnN 3 meta-omics only available via pip
python3 -m pip install humann --no-binary :all:
python3 -m pip install metaphlan --no-binary :all:
humann_config --update database_folders nucleotide /unlock/references/databases/HUMAnN/chocophlan
humann_config --update database_folders utility_mapping /unlock/references/databases/HUMAnN/utility_mapping
humann_config --update database_folders protein /unlock/references/databases/HUMAnN/uniref
## Pip dependencies
python3 -m pip install wheel biom-format pysam numpy matplotlib checkm-genome gtdbtk
python3 -m pip install wheel biom-format pysam numpy matplotlib checkm-genome gtdbtk rdflib python-irodsclient html5lib
python3 -m pip install cwltool==3.1.20211004060744 && \
cwltool --version && python3 -m pip install html5lib
python3 -m pip install cwltool==3.1.20211004060744
cwltool --version && python3 -m pip install html5lib
## Pip quast rc1 for 3.9 support
wget https://github.com/ablab/quast/releases/download/quast_5.1.0rc1/quast-5.1.0rc1.tar.gz &&\
pip install quast-5.1.0rc1.tar.gz
wget https://github.com/ablab/quast/releases/download/quast_5.1.0rc1/quast-5.1.0rc1.tar.gz
pip install quast-5.1.0rc1.tar.gz
## CheckM
checkm data setRoot /unlock/references/databases/CheckM/
# R packages
R -e "install.packages('docopt',dependencies=TRUE, repos='https://packagemanager.rstudio.com/all/__linux__/bionic/latest')"
Rscript /scripts/install2.R -n 10 -s -l /unlock/infrastructure/R/library -r https://packagemanager.rstudio.com/all/__linux__/bionic/latest data.table futile.logger ggplot2 optparse plyr readr reshape2 scales viridis yaml parallel
\ No newline at end of file
Rscript /scripts/install2.R -n 10 -s -l /unlock/infrastructure/R/library -r https://packagemanager.rstudio.com/all/__linux__/bionic/latest data.table futile.logger ggplot2 optparse plyr readr reshape2 scales viridis yaml parallel castor
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment