用eclipse 连接kafka streaming 执行worldcount

最新推荐文章于 2019-12-10 18:36:10 发布

原创最新推荐文章于 2019-12-10 18:36:10 发布 · 478 阅读

0 ·

本内容遵循CC 4.0 BY-SA版权协议

收录于

经验

hadoop

大数据+机器学习+oracle

本文介绍了一个使用 Apache Spark Streaming 进行 Kafka 数据流处理的示例程序。该程序从 Kafka 中读取数据，执行单词计数，并展示如何配置 Spark Streaming 上下文。

package com.ecloud;

import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Pattern;

import org.apache.spark.SparkConf;
import org.apache.spark.streaming.Duration;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaPairDStream;
import org.apache.spark.streaming.api.java.JavaPairReceiverInputDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import org.apache.spark.streaming.kafka.KafkaUtils;

import scala.Tuple2;

public class Kafka_stream_test {

private static final Pattern SPACE = Pattern.compile(" ");

public static void main(String[] args) throws Exception {
SparkConf sparkConf = new SparkConf().setMaster("local").setAppName("JavaKafkaWordCount");
// Create the context with 2 seconds batch size
JavaStreamingContext jssc = new JavaStreamingContext(sparkConf, new Duration(2000));

// JavaStreamingContext jssc = new JavaStreamingContext(sparkConf, Durations.seconds(2));

int numThreads = Integer.parseInt("3");
Map<String, Integer> topicMap = new HashMap<>();
// String[] topics = args[2].split(",");
// for (String topic: topics) {
String topic ="kafkatopic8";
topicMap.put(topic, numThreads);
// }

JavaPairReceiverInputDStream<String, String> messages =
KafkaUtils.createStream(jssc, "manger:2181", "test-kafka", topicMap);

JavaDStream<String> lines = messages.map(Tuple2::_2);

JavaDStream<String> words = lines.flatMap(x -> Arrays.asList(SPACE.split(x)).iterator());

JavaPairDStream<String, Integer> wordCounts = words.mapToPair(s -> new Tuple2<>(s, 1))
.reduceByKey((i1, i2) -> {System.out.println(i1+i2);return i1 + i2;});

wordCounts.foreachRDD(x->System.out.println(x.count()));
jssc.start();
jssc.awaitTermination();
}

}