Skip to content
Snippets Groups Projects
start-spark 401 B
Newer Older
Dawes's avatar
Dawes committed
export SPARK_LOG_DIR=${HOME}/.spark/${SLURM_JOB_ID}
mkdir -p $SPARK_LOG_DIR
unlink ${HOME}/.spark/current &>/dev/null
ln -s $SPARK_LOG_DIR ${HOME}/.spark/current
Dawes's avatar
Dawes committed
export SPARK_CONF_DIR=$SPARK_LOG_DIR/conf
mkdir -p $SPARK_CONF_DIR
Dawes's avatar
Dawes committed

WURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
srun $WURDIR/start-spark-executors.sh &
Dawes's avatar
Dawes committed

while [ ! -s $SPARK_CONF_DIR/spark-defaults.conf ] ; do
  sleep 1
done