Spark

export PYSPARK_PYTHON='/usr/bin/python3';\
export SPARK_HOME='/opt/cli/spark-3.3.0-bin-hadoop3';\
export JAVA_HOME='/usr/lib/jvm/java-17-openjdk-amd64';\
export PATH=$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin

spark-shell
pyspark

http://localhost:8080/
http://localhost:7077/
http://localhost:4040/

ssh -L 8080:localhost:8080 [email protected]
ssh -L 7077:localhost:7077 [email protected]

Master Node

sudo apt -qq update;\
export PYSPARK_PYTHON='/usr/bin/python3';\
export SPARK_HOME='/opt/cli/spark-3.3.0-bin-hadoop3';\
export JAVA_HOME='/usr/lib/jvm/java-17-openjdk-arm64';\
bash <(curl -s 'https://cdn.chorke.org/exec/cli/bash/install/apache-spark-master/3.3.0.sh.txt')

sudo systemctl daemon-reload
sudo systemctl enable spark-master.service
sudo systemctl start  spark-master.service
sudo systemctl status spark-master.service

Worker Node

sudo apt -qq update;\
export PYSPARK_PYTHON='/usr/bin/python3';\
export SPARK_MASTER='spark://ns12-pc04:7077';\
export SPARK_HOME='/opt/cli/spark-3.3.0-bin-hadoop3';\
export JAVA_HOME='/usr/lib/jvm/java-17-openjdk-amd64';\
bash <(curl -s 'https://cdn.chorke.org/exec/cli/bash/install/apache-spark-slave/3.3.0.sh.txt')

sudo systemctl daemon-reload
sudo systemctl enable spark-slave.service
sudo systemctl start  spark-slave.service
sudo systemctl status spark-slave.service

Knowledge

ssh-keygen -b 4096 -t rsa -f ~/.ssh/id_rsa -q -N "spark@${HOSTNAME}" readlink -f /usr/bin/java | sed "s:bin/java::" sudo apt-get install pdsh sudo apt-get install ssh
su -h spark sudo -u spark -H sh -c "whoami; echo ${HOME}" sh $SPARK_HOME/bin/spark-shell sh $SPARK_HOME/bin/pyspark http://127.0.0.1:8080 http://127.0.0.1:8088 http://127.0.0.1:9870 http://127.0.0.1:4040
sudo apt dist-upgrade sudo do-release-upgrade sudo apt --fix-broken install sudo apt install ubuntu-desktop		[Service] User=spark Group=spark Type=forking SuccessExitStatus=143
if [ -f '/etc/os-release' ];then HOST_OS_ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') HOST_OS_ID_LIKE=$(grep -oP '(?<=^ID_LIKE=).+' /etc/os-release | tr -d '"') HOST_OS_VERSION=$(grep -oP '(?<=^VERSION_ID=).+' /etc/os-release | tr -d '"') fi

References

Spark » Systemd » Failed to stop Master/Slave Node Spark » Download » Archive Spark » Change WebUI Port Spark » SQL & DataFrames Spark » Machine Learning Spark » Streaming Spark » Download Spark » Examples Spark » GraphX Spark » Install	Spark » Concatenate columns in DataFrame Spark » Structured Streaming Programming Spark » Apache Spark, Hive & Spring Boot Spark » Integration Testing Example Spark » Install Master as a Daemon Spark » Install Slave as a Daemon Spark » As a Linux Service Spark » Apache Camel Spark » Introduction Spark » Spring Boot	KillSignal interact with TimeoutStopSec Delay Systemd Service If File Exist System Unit Conditions & Asserts VS Code on iPad Pro Apache Camel Mosquitto ActiveMQ Keycloak GraphQL Grafana
Spark » Running Spark on Top of a Hadoop YARN Spark » Deploy » Cluster Mode Overview Spark » Deploy » Standalone Mode Spark » Deploy » Spark Submit Spark » Deploy » Kubernetes Spark » Multi-Node Cluster Spark » Deploy » Cluster Spark » Deploy » Mesos Spark » Deploy » YARN	Spark » Unable to load native-hadoop library Spark » Apache Arrow Java Cookbook Spark » Access remote HDFS Spark » API Docs » Python Spark » API Docs » Scala Spark » API Docs » Java Spark » API Docs » SQL Spark » API Docs » R	AWS » Bastion Host SSH Tunneling Set Up Environment for Spark Machine Learning Linux Containers Apache Hive Aerospike Hadoop Jupyter NLP
Systemd » Safe Remove Services Bastion SSH Tunneling