Newer
Older
export SPARK_LOG_DIR=${HOME}/.spark/${SLURM_JOB_ID}
mkdir -p $SPARK_LOG_DIR
unlink ${HOME}/.spark/current &>/dev/null
ln -s $SPARK_LOG_DIR ${HOME}/.spark/current
WURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
srun $WURDIR/start-spark-executors.sh &