start-spark

export SPARK_LOG_DIR=${HOME}/.spark/${SLURM_JOB_ID}
mkdir -p $SPARK_LOG_DIR
unlink ${HOME}/.spark/current &>/dev/null
ln -s $SPARK_LOG_DIR ${HOME}/.spark/current
export SPARK_CONF_DIR=$SPARK_LOG_DIR/conf
mkdir -p $SPARK_CONF_DIR

WURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
srun $WURDIR/start-spark-executors.sh &

while [ ! -s $SPARK_CONF_DIR/spark-defaults.conf ] ; do
  sleep 1
done