Spark

Spark Master

sudo tee -a /etc/systemd/system/spark-master.service >/dev/null <<EOF
[Unit]
Description=Apache Spark Master
Documentation=https://spark.apache.org/docs/3.3.0
Wants=network-online.target
After=network-online.target
After=systemd-user-sessions.service

[Service]
User=spark
Group=spark
Type=forking
EnvironmentFile=${SPARK_HOME}/etc/env
ExecStart=${SPARK_HOME}/sbin/start-master.sh
ExecStop=${SPARK_HOME}/sbin/stop-master.sh

[Install]
WantedBy=multi-user.target
EOF

Slave Daemon

sudo tee -a /etc/systemd/system/spark-slave.service >/dev/null <<EOF
[Unit]
Description=Apache Spark Slave
Documentation=https://spark.apache.org/docs/3.3.0
Wants=network-online.target
After=network-online.target
AssertFileNotEmpty=${SPARK_HOME}/etc/env

[Service]
User=spark
Group=spark
Type=forking

EnvironmentFile=${SPARK_HOME}/etc/env
WorkingDirectory={{SPARK_HOME}}/sbin
ExecStartPre=/bin/bash -c "if [ -z \"\${PYSPARK_PYTHON}\" ]; then echo \"Variable PYSPARK_PYTHON not set in ${SPARK_HOME}/etc/env\"; exit 1; fi"
ExecStartPre=/bin/bash -c "if [ -z \"\${SPARK_HOME}\" ]; then echo \"Variable SPARK_HOME not set in ${SPARK_HOME}/etc/env\"; exit 1; fi"
ExecStartPre=/bin/bash -c "if [ -z \"\${JAVA_HOME}\" ]; then echo \"Variable JAVA_HOME not set in ${SPARK_HOME}/etc/env\"; exit 1; fi"
ExecStartPost=/bin/bash -c "echo \$MAINPID > ${SPARK_HOME}/etc/spark-slave.pid"
ExecStart={{SPARK_HOME}}/sbin/start-slave.sh spark://10.20.22.10:7077
ExecStop={{SPARK_HOME}}/sbin/stop-slave.sh
Restart=on-failure
RestartSec=10s

[Install]
WantedBy=multi-user.target
EOF

References

Spark » Download » Archive Spark » SQL & DataFrames Spark » Machine Learning Spark » Streaming Spark » Download Spark » Examples Spark » GraphX Hadoop Jupyter NLP	Spark » Structured Streaming Programming Spark » Apache Spark, Hive & Spring Boot Spark » Integration Testing Example Spark » Install Master as a Daemon Spark » Install Slave as a Daemon Spark » As a Linux Service VS Code on iPad Pro Machine Learning Apache Hive Aerospike	Delay Systemd Service If File Exist System Unit Conditions & Asserts Keycloak GraphQL Grafana
Spark » Deploy » Cluster Mode Overview Spark » Deploy » Standalone Mode Spark » Deploy » Spark Submit Spark » Deploy » Kubernetes Spark » Deploy » Cluster Spark » Deploy » Mesos Spark » Deploy » YARN	Spark » API Docs » Python Spark » API Docs » Scala Spark » API Docs » Java Spark » API Docs » SQL Spark » API Docs » R