Spark: Difference between revisions
Jump to navigation
Jump to search
| Line 52: | Line 52: | ||
* [https://spark.apache.org/sql/ Spark » SQL & DataFrames] | * [https://spark.apache.org/sql/ Spark » SQL & DataFrames] | ||
* [https://spark.apache.org/mllib/ Spark » Machine Learning] | * [https://spark.apache.org/mllib/ Spark » Machine Learning] | ||
* [https://stackoverflow.com/questions/36083783/ Spark » Change WebUI Port] | |||
* [https://spark.apache.org/streaming/ Spark » Streaming] | * [https://spark.apache.org/streaming/ Spark » Streaming] | ||
* [https://spark.apache.org/downloads.html Spark » Download] | * [https://spark.apache.org/downloads.html Spark » Download] | ||
| Line 57: | Line 58: | ||
* [https://spark.apache.org/graphx/ Spark » GraphX] | * [https://spark.apache.org/graphx/ Spark » GraphX] | ||
* [https://cdn.chorke.org/exec/cli/bash/install/ Spark » Install] | * [https://cdn.chorke.org/exec/cli/bash/install/ Spark » Install] | ||
| valign="top" | | | valign="top" | | ||
| Line 109: | Line 109: | ||
* [[Hadoop]] | * [[Hadoop]] | ||
* [[Jupyter]] | * [[Jupyter]] | ||
* [[NLP]] | |||
|} | |} | ||
Revision as of 15:04, 26 September 2022
export PYSPARK_PYTHON='/usr/bin/python3';\
export SPARK_HOME='/opt/cli/spark-3.3.0-bin-hadoop3';\
export JAVA_HOME='/usr/lib/jvm/java-17-openjdk-amd64';\
export PATH=$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin
spark-shell pyspark
http://localhost:8080/ http://localhost:7077/ http://localhost:4040/
ssh -L 8080:localhost:8080 [email protected] ssh -L 7077:localhost:7077 [email protected]
Master Node
sudo apt -qq update;\
export PYSPARK_PYTHON='/usr/bin/python3';\
export SPARK_HOME='/opt/cli/spark-3.3.0-bin-hadoop3';\
export JAVA_HOME='/usr/lib/jvm/java-17-openjdk-arm64';\
bash <(curl -s 'https://cdn.chorke.org/exec/cli/bash/install/apache-spark-master/3.3.0.sh.txt')
sudo systemctl daemon-reload sudo systemctl enable spark-master.service sudo systemctl start spark-master.service sudo systemctl status spark-master.service
Worker Node
sudo apt -qq update;\
export PYSPARK_PYTHON='/usr/bin/python3';\
export SPARK_MASTER='spark://ns12-pc04:7077';\
export SPARK_HOME='/opt/cli/spark-3.3.0-bin-hadoop3';\
export JAVA_HOME='/usr/lib/jvm/java-17-openjdk-amd64';\
bash <(curl -s 'https://cdn.chorke.org/exec/cli/bash/install/apache-spark-slave/3.3.0.sh.txt')
sudo systemctl daemon-reload sudo systemctl enable spark-slave.service sudo systemctl start spark-slave.service sudo systemctl status spark-slave.service