spark任务运行

powershell 复制代码

在这里插入代码片
[root@hadoop000 conf]# java -version
java version "1.8.0_144"
Java(TM) SE Runtime Environment (build 1.8.0_144-b01)

powershell 复制代码

[root@hadoop000 conf]# echo $JAVA_HOME
/home/hadoop/app/jdk1.8.0_144

powershell 复制代码

[root@hadoop000 conf]# vi spark-env.sh
[root@hadoop000 conf]# mv slaves.template slaves
[root@hadoop000 conf]# vi slave

powershell 复制代码

/home/hadoop/app/spark-2.2.0-bin-2.6.0-cdh5.7.0/bin/spark-submit \
--class org.apache.spark.examples.SparkPi \
--master spark://192.168.2.111:7077 \
--executor-memory 1G \
--total-executor-cores 2 \
/home/hadoop/app/spark-2.2.0-bin-2.6.0-cdh5.7.0/examples/jars/spark-examples_2.11-2.2.0.jar

powershell 复制代码

[root@hadoop000 bin]# /home/hadoop/app/spark-2.2.0-bin-2.6.0-cdh5.7.0/bin/spark-shell \
> --master spark://192.168.2.111:7077 \
> --executor-memory 2G \
> --total-executor-cores 2
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
Setting default log level to "WARN".
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
25/02/15 16:45:37 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
25/02/15 16:45:43 WARN ObjectStore: Failed to get database global_temp, returning NoSuchObjectException
Spark context Web UI available at http://192.168.2.111:4040
Spark context available as 'sc' (master = spark://192.168.2.111:7077, app id = app-20250215164538-0002).
Spark session available as 'spark'.
Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ `/ __/  '_/
   /___/ .__/\_,_/_/ /_/\_\   version 2.2.0
      /_/
         
Using Scala version 2.11.8 (Java HotSpot(TM) 64-Bit Server VM, Java 1.8.0_144)
Type in expressions to have them evaluated.
Type :help for more information.

powershell 复制代码

scala> sc.textFile("hdfs://192.168.2.102:9000//user/spark/input/word.txt").flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_).saveAsTextFile("hdfs://192.168.2.102:9000//user/spark/out")

powershell 复制代码

scala> sc.textFile("hdfs://192.168.2.102:9000/user/spark/out/*").collect().foreach(println)
(orange,1)
(queen,1)
(rabbit,1)
(fish,1)
(dog,1)
(apple,1)
(pig,1)
(umbrella,1)
(snake,1)
(lion,1)
(juice,1)
(cat,1)
(tiger,1)
(banana,1)
(monkey,1)
(nose,1)
(kite,1)
(elephant,1)
(ice,1)
(goat,1)
(horse,1)

运行环境

步骤

spark-shell