start-spark

export SPARK_LOG_DIR=${HOME}/.spark/${SLURM_JOB_ID}
mkdir -p $SPARK_LOG_DIR
mkdir -p $SPARK_LOG_DIR/conf
export SPARK_CONF_DIR=$SPARK_LOG_DIR/conf

srun ./start-spark-executors.sh &

while [ ! -s $SPARK_CONF_DIR/spark-defaults.conf ] ; do
  sleep 1
done