spark的wordcount示例:
scala
package spark
import org.apache.spark._
object TestSparkWordCount {
def main(args: Array[String]): Unit = {
val sc = new SparkContext(new SparkConf().setAppName("wordcount").setMaster("local[*]"))
// hello scala
// hello spark
val rdd = sc.textFile("src/main/resources/test.txt")
val wordCount = rdd.flatMap(line => line.split(" "))
.map(word => (word, 1))
.reduceByKey((a, b) => (a + b))
wordCount.foreach(println)
// (spark,1)
// (scala,1)
// (hello,2)
}
}
flink的wordcount示例:
scala
package com.pinko.testcase
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.api.scala._
object StreamWordCount {
def main(args: Array[String]): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
env.setParallelism(1)
// hello world
// hello flink
val text = env.readTextFile("src/main/resources/output/1.txt")
val result = text.flatMap(_.split(" "))
.map((_, 1))
.keyBy(0)
.sum(1)
result.print("result")
env.execute("StreamWordCount")
// result> (hello,1)
// result> (world,1)
// result> (hello,2)
// result> (flink,1)
}
}