Spark Streaming 整合Kafka的 Offset 管理 【数据零丢失之 checkpoint 方式管理Offset】

import kafka.serializer.StringDecoder
import org.apache.spark.SparkConf
import org.apache.spark.streaming.kafka.KafkaUtils
import org.apache.spark.streaming.{Duration, Seconds, StreamingContext}

/**
  * Description: offset : checkpoint ==>meta write to hdfs
  *
  * 使用的0.8版本  spark-streaming-kafka-0-8_2.11
  *
  * checkpoint缺点：采用这种方式管理offset ,当你的业务逻辑代码发生变化，你的checkpoint 就无法使用到了
  *
  * @Author: 留歌36
  * @Date: 2019/8/15 10:40
  */
object Offset02App {
  def main(args: Array[String]): Unit ={
    val conf = new SparkConf().setAppName("Offset02App").setMaster("local[2]")
    val kafkaParams = Map[String, String](
      "metadata.broker.list"->"xxx:9092",
      "auto.offset.reset" -> "smallest"
    )
    val topics = "exactlyonce_topic".split(",").toSet
    val checkpointDirectory = "hdfs://xxx:8020/offset"

    /**
      * hadoop fs -mkdir /offset
      *  hadoop fs -ls /offset
      */

  // Function to create and setup a new StreamingContext
    def functionToCreateContext(): StreamingContext = {
      val ssc = new StreamingContext(conf, Seconds(10))   // new context

      val stream = KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder](ssc, kafkaParams, topics)


      // 设置checkpoint
      ssc.checkpoint(checkpointDirectory)   // set checkpoint directory
      stream.checkpoint(Duration(10*1000)) // 每隔10s 做一次checkpoint提交

      // 业务逻辑
      stream.foreachRDD(rdd => {
        if (!rdd.isEmpty()) {
          println(s"留歌${rdd.count()}条数据" )
        }
      })

      ssc
    }

    // Get StreamingContext from checkpoint data or create a new one
    val ssc = StreamingContext.getOrCreate(checkpointDirectory, functionToCreateContext _)


    ssc.start() // 启动程序
    ssc.awaitTermination() // 等待程序终止
  }

}