Spark: Difference between revisions

From Chorke Wiki
Jump to navigation Jump to search
Line 48: Line 48:
{|
{|
| valign="top" |
| valign="top" |
* [https://stackoverflow.com/questions/73853179/ Spark » Systemd » Failed to stop Master/Slave Node]
* [https://archive.apache.org/dist/spark/ Spark » Download » Archive]
* [https://archive.apache.org/dist/spark/ Spark » Download » Archive]
* [https://spark.apache.org/sql/ Spark » SQL & DataFrames]
* [https://spark.apache.org/sql/ Spark » SQL & DataFrames]

Revision as of 05:06, 26 September 2022

export PYSPARK_PYTHON='/usr/bin/python3';\
export SPARK_HOME='/opt/cli/spark-3.3.0-bin-hadoop3';\
export JAVA_HOME='/usr/lib/jvm/java-17-openjdk-amd64';\
export PATH=$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin

spark-shell
pyspark

http://localhost:8080/
http://localhost:7077/
http://localhost:4040/

ssh -L 8080:localhost:8080 [email protected]
ssh -L 7077:localhost:7077 [email protected]

Master Node

sudo apt -qq update;\
export PYSPARK_PYTHON='/usr/bin/python3';\
export SPARK_HOME='/opt/cli/spark-3.3.0-bin-hadoop3';\
export JAVA_HOME='/usr/lib/jvm/java-17-openjdk-arm64';\
bash <(curl -s 'https://cdn.chorke.org/exec/cli/bash/install/apache-spark-master/3.3.0.sh.txt')

sudo systemctl daemon-reload
sudo systemctl enable spark-master.service
sudo systemctl start  spark-master.service
sudo systemctl status spark-master.service

Worker Node

sudo apt -qq update;\
export PYSPARK_PYTHON='/usr/bin/python3';\
export SPARK_MASTER='spark://ns12-pc04:7077';\
export SPARK_HOME='/opt/cli/spark-3.3.0-bin-hadoop3';\
export JAVA_HOME='/usr/lib/jvm/java-17-openjdk-amd64';\
bash <(curl -s 'https://cdn.chorke.org/exec/cli/bash/install/apache-spark-slave/3.3.0.sh.txt')

sudo systemctl daemon-reload
sudo systemctl enable spark-slave.service
sudo systemctl start  spark-slave.service
sudo systemctl status spark-slave.service

References

Retrieved from "https://cdn.chorke.org/weke/index.php?title=Spark&oldid=8415"