Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
UNLOCK
docker
Commits
128227fb
Commit
128227fb
authored
Jan 07, 2022
by
Jasper Koehorst
Browse files
setup improvements
parent
48ff859d
Changes
3
Hide whitespace changes
Inline
Side-by-side
kubernetes/Dockerfile
View file @
128227fb
...
...
@@ -118,9 +118,9 @@ COPY scripts /scripts
COPY
conda /conda
# Set different location for R packages
COPY
scripts/Renviron /root/.Renviron
#
COPY scripts/Renviron /root/.Renviron
# Extend bashrc
RUN
echo
"export R_LIBS=/unlock/infrastructure/R/library/"
>>
~/.bashrc
RUN
echo
"export R_LIBS_SITE=/unlock/infrastructure/R/library/"
>>
~/.bashrc
RUN
echo
"export R_LIBS_USER=/unlock/infrastructure/R/library/root@3b440897a84b"
>>
~/.bashrc
\ No newline at end of file
RUN
echo
"export R_LIBS=/unlock/infrastructure/R/library"
>>
~/.bashrc
RUN
echo
"export R_LIBS_SITE=/unlock/infrastructure/R/library"
>>
~/.bashrc
RUN
echo
"export R_LIBS_USER=/unlock/infrastructure/R/library"
>>
~/.bashrc
\ No newline at end of file
kubernetes/run.sh
View file @
128227fb
...
...
@@ -56,7 +56,7 @@ python3 /scripts/metadata.py -key cwl -value $cwl -unit running -add -file $yaml
[
$?
-eq
0
]
&&
echo
"Metadata added"
||
exit
1
# Download latest yaml file
java
-jar
/unlock/infrastructure/binaries/irods/IRODSTransfer.jar
-pull
--files
$yaml
--local
$yaml_dir
--force
java
-jar
/unlock/infrastructure/binaries/irods/IRODSTransfer.jar
-pull
--files
$yaml
--local
$yaml_dir
[
$?
-eq
0
]
&&
echo
"downloaded
$yaml
"
||
exit
1
# Obtain destination path
...
...
@@ -67,27 +67,18 @@ grep "[0-9]\+_irods:" $yaml | awk '{print $2}' | sort | uniq | xargs dirname | x
# Download run files from iRODS
grep
"[0-9]
\+
_irods:"
$yaml
|
awk
'{print $2}'
|
sort
|
uniq
|
awk
'{print "java -jar /unlock/infrastructure/binaries/irods/IRODSTransfer.jar --pull --local "$1" --files "$1" --force"}'
| sh
# Start a conda environment when needed
conda_env
=
`
grep
conda
$yaml
|
awk
-F
": "
'{print $2}'
`
if
!
[[
-z
"
$conda_env
"
]]
;
then
source
/root/miniconda/bin/activate
&&
conda activate
$conda_env
fi
# Run workflow with or without provenance
if
$prov
;
then
cwltool
--tmpdir-prefix
/unlock/rancher/unlockcwl
--no-container
--preserve-entire-environment
--outdir
$destination_path
--provenance
$destination_path
/PROVENANCE
$cwl
$yaml
cwltool
--cachedir
CACHE
--tmpdir-prefix
/unlock/rancher/unlockcwl
--no-container
--preserve-entire-environment
--outdir
$destination_path
--provenance
$destination_path
/PROVENANCE
$cwl
$yaml
else
cwltool
--tmpdir-prefix
/unlock/rancher/unlockcwl
--no-container
--preserve-entire-environment
--outdir
$destination_path
$cwl
$yaml
cwltool
--cachedir
CACHE
--tmpdir-prefix
/unlock/rancher/unlockcwl
--no-container
--preserve-entire-environment
--outdir
$destination_path
$cwl
$yaml
fi
# Check exit status of cwl
exit_status
=
$?
# Start a conda environment when needed
conda_env
=
`
grep
conda
$yaml
|
awk
-F
": "
'{print $2}'
`
if
!
[[
-z
"
$conda_env
"
]]
;
then
source
/root/miniconda/bin/activate
&&
conda deactivate
fi
# Make sure conda is deactivated afterwards in addition cwl scripts should do this on their own
source
/root/miniconda/bin/activate
&&
conda deactivate
# When success, update avu on yaml file and upload results
if
[
$exit_status
-eq
0
]
;
then
...
...
kubernetes/scripts/setup.sh
View file @
128227fb
...
...
@@ -3,34 +3,32 @@
# Ensures that R can use the library folder
mkdir
--parents
/unlock/infrastructure/R/library
################################################################
# Conda environments
################################################################
# Update conda
conda update
-y
-n
base
-c
defaults conda
## Conda environment for picrust
conda update
-y
-n
base conda
# picrust
source
/root/miniconda/bin/activate
&&
\
cd
/unlock/infrastructure/binaries/picrust2-2.4.1/
&&
\
conda
env
create
-f
/conda/picrust2_env.yml
-p
/unlock/infrastructure/conda/picrust2
&&
\
conda activate /unlock/infrastructure/conda/picrust2
&&
\
pip
install
--editable
.
&&
\
# Needed?
conda
env
update
-p
/unlock/infrastructure/conda/picrust2
--file
/conda/picrust2_env.yml
&&
\
conda deactivate
# New attempt for picrust
source
/root/miniconda/bin/activate
mamba create
--yes
--prefix
/unlock/infrastructure/conda/picrust2
-c
bioconda
-c
conda-forge
picrust2
=
2.4.2
# Krona
mamba create
--yes
--prefix
/unlock/infrastructure/conda/krona
&&
\
mamba
install
--yes
-c
bioconda krona
&&
\
ktUpdateTaxonomy.sh
mamba create
--yes
--prefix
/unlock/infrastructure/conda/krona
mamba
install
--yes
-c
bioconda krona
ktUpdateTaxonomy.sh
# Medaka
bash
source
/root/miniconda/bin/activate
mamba create
--yes
--prefix
/unlock/infrastructure/conda/medaka
-c
bioconda
-c
conda-forge
medaka
=
1.5.0
# conda activate /unlock/infrastructure/conda/medaka
# mamba install --yes -c bioconda -c conda-forge medaka=1.0.3
################################################################
# Pip environment
################################################################
# Python virtual environment
python3
-m
venv /unlock/infrastructure/venv
...
...
@@ -39,26 +37,26 @@ source /unlock/infrastructure/venv/bin/activate
# Upgrade because we can
/unlock/infrastructure/venv/bin/python3
-m
pip
install
--upgrade
pip
## HUMAnN meta-omics
python3
-m
pip
install
humann
--no-binary
:all:
&&
\
python3
-m
pip
install
metaphlan
--no-binary
:all:
&&
\
humann_config
--update
database_folders nucleotide /unlock/references/databases/HUMAnN/chocophlan
&&
\
humann_config
--update
database_folders utility_mapping /unlock/references/databases/HUMAnN/utility_mapping
&&
\
humann_config
--update
database_folders protein /unlock/references/databases/HUMAnN/uniref
## HUMAnN
3
meta-omics
only available via pip
python3
-m
pip
install
humann
--no-binary
:all:
python3
-m
pip
install
metaphlan
--no-binary
:all:
humann_config
--update
database_folders nucleotide /unlock/references/databases/HUMAnN/chocophlan
humann_config
--update
database_folders utility_mapping /unlock/references/databases/HUMAnN/utility_mapping
humann_config
--update
database_folders protein /unlock/references/databases/HUMAnN/uniref
## Pip dependencies
python3
-m
pip
install
wheel biom-format pysam numpy matplotlib checkm-genome gtdbtk
python3
-m
pip
install
wheel biom-format pysam numpy matplotlib checkm-genome gtdbtk
rdflib python-irodsclient html5lib
python3
-m
pip
install
cwltool
==
3.1.20211004060744
&&
\
cwltool
--version
&&
python3
-m
pip
install
html5lib
python3
-m
pip
install
cwltool
==
3.1.20211004060744
cwltool
--version
&&
python3
-m
pip
install
html5lib
## Pip quast rc1 for 3.9 support
wget https://github.com/ablab/quast/releases/download/quast_5.1.0rc1/quast-5.1.0rc1.tar.gz
&&
\
pip
install
quast-5.1.0rc1.tar.gz
wget https://github.com/ablab/quast/releases/download/quast_5.1.0rc1/quast-5.1.0rc1.tar.gz
pip
install
quast-5.1.0rc1.tar.gz
## CheckM
checkm data setRoot /unlock/references/databases/CheckM/
# R packages
R
-e
"install.packages('docopt',dependencies=TRUE, repos='https://packagemanager.rstudio.com/all/__linux__/bionic/latest')"
Rscript /scripts/install2.R
-n
10
-s
-l
/unlock/infrastructure/R/library
-r
https://packagemanager.rstudio.com/all/__linux__/bionic/latest data.table futile.logger ggplot2 optparse plyr readr reshape2 scales viridis yaml parallel
\ No newline at end of file
Rscript /scripts/install2.R
-n
10
-s
-l
/unlock/infrastructure/R/library
-r
https://packagemanager.rstudio.com/all/__linux__/bionic/latest data.table futile.logger ggplot2 optparse plyr readr reshape2 scales viridis yaml parallel castor
\ No newline at end of file
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment