Newer
Older
#!/bin/bash
export SPARK_LOG_DIR=${HOME}/.spark/${SLURM_JOB_ID}
export SPARK_CONF_DIR=$SPARK_LOG_DIR/conf
if [ "$SLURM_CPUS_PER_TASK" = "" ]; then
SLURM_CPUS_PER_TASK=1
fi
CPU=$SLURM_CPUS_PER_TASK
if [ "$SLURM_MEM_PER_CPU" = "" ]; then
MEM=$(( $SLURM_MEM_PER_NODE / $SLURM_CPUS_ON_NODE * $SLURM_CPUS_PER_TASK ))
else
MEM=$(( $SLURM_MEM_PER_CPU * $SLURM_CPUS_PER_TASK ))
fi
if [ "$SPARK_WORKER_WEBUI_PORT" = "" ]; then
SPARK_WORKER_WEBUI_PORT=8081
fi
MASTER=$(cat $SPARK_LOG_DIR/master)
WORKER_NUM=$(( $SLURM_PROCID + 1 ))
PORT_NUM=$(( $SPARK_WORKER_PORT + $WORKER_NUM - 1 ))
WEBUI_PORT=$(( $SPARK_WORKER_WEBUI_PORT + $WORKER_NUM - 1 ))
WORKER_LOG=$SPARK_LOG_DIR/worker-${WORKER_NUM}.out
WORKER_WORKDIR=$SPARK_LOG_DIR/work
$SPARK_HOME/bin/spark-class org.apache.spark.deploy.worker.Worker --memory ${MEM}M --cores $CPU --work-dir $WORKER_WORKDIR \
--webui-port $WEBUI_PORT "$MASTER" >> "$WORKER_LOG" 2>&1 < /dev/null