天天看点

scala sparkstreaming wordcount

import org.apache.spark.SparkConf

import org.apache.spark.streaming.StreamingContext

import org.apache.spark.streaming.Durations

object WordCountStream {

  def main(args: Array[String]): Unit = {

    val conf = new SparkConf().setAppName("WordCount").setMaster("local[2]")

    val jsc = new StreamingContext(conf,Durations.seconds(5))

    val lines = jsc.textFileStream("D:\\BigData\\mockData\\mockData.txt")

    val words = lines.flatMap(_.split(" "))

    val pairs = words.map{(_,1)}

    val result = pairs.reduceByKey(_ + _)

    result.print()

    jsc.start()

    jsc.awaitTermination()

    jsc.stop()

  }

}

继续阅读