start-slave.sh

#!/bin/bash 


export SPARK_LOG_DIR=${HOME}/.spark/${SLURM_JOB_ID}
export SPARK_CONF_DIR=$SPARK_LOG_DIR/conf

if [ "$SLURM_CPUS_PER_TASK" = "" ]; then
  SLURM_CPUS_PER_TASK=1
fi
CPU=$SLURM_CPUS_PER_TASK
if [ "$SLURM_MEM_PER_CPU" = "" ]; then
  MEM=$(( $SLURM_MEM_PER_NODE / $SLURM_CPUS_ON_NODE * $SLURM_CPUS_PER_TASK ))
else
  MEM=$(( $SLURM_MEM_PER_CPU * $SLURM_CPUS_PER_TASK ))
fi

if [ "$SPARK_WORKER_WEBUI_PORT" = "" ]; then
  SPARK_WORKER_WEBUI_PORT=8081
fi

MASTER=$(cat $SPARK_LOG_DIR/master)
WORKER_NUM=$(( $SLURM_PROCID + 1 ))
PORT_NUM=$(( $SPARK_WORKER_PORT + $WORKER_NUM - 1 ))
WEBUI_PORT=$(( $SPARK_WORKER_WEBUI_PORT + $WORKER_NUM - 1 ))
WORKER_LOG=$SPARK_LOG_DIR/worker-${WORKER_NUM}.out

WORKER_WORKDIR=$SPARK_LOG_DIR/work

$SPARK_HOME/bin/spark-class org.apache.spark.deploy.worker.Worker --memory ${MEM}M --cores $CPU --work-dir $WORKER_WORKDIR \
  --webui-port $WEBUI_PORT "$MASTER" >> "$WORKER_LOG" 2>&1 < /dev/null