hadoop 3.x大数据集群搭建系列4-安装Spark

一. 下载spark和scala并解压

cd /home/software
wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz --no-check-certificate

tar -xvf spark-3.3.1-bin-hadoop3.tgz -C /home/

cd /home/software
官网下载scala:https://www.scala-lang.org/download/2.13.1.html

tar -xvf scala-2.13.1.tgz -C /home

二. 配置SPARK_HOME环境变量

vi /etc/profile
# 添加如下内容
export SPARK_HOME=/home/spark-3.3.1-bin-hadoop3
export PATH=$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin

export SCALA_HOME=/home/scala-2.13.1
export PATH= $PATH:$SCALA_HOME/bin


# 加载使其生效
source /etc/profile

三. 修改配置

# 进入spark conf目录
cd /home/spark-3.3.1-bin-hadoop3/conf
# 拷贝模板文件
cp spark-defaults.conf.template spark-defaults.conf
cp spark-env.sh.template spark-env.sh

3.1 修改 spark-defaults.conf

vim spark-defaults.conf
# 添加如下内容
spark.master                     spark://hp5:7077
spark.serializer                 org.apache.spark.serializer.KryoSerializer
spark.driver.memory              512m
spark.executor.memory            512m

3.2 修改spark-env.sh

export JAVA_HOME=/usr/java/jdk1.8.0_181
export HADOOP_HOME=/home/hadoop-3.3.2
export HADOOP_CONF_DIR=/home/hadoop-3.3.