大数据中各种WordCount实现

前言：

• Spark编程惯例
• Spark编程的入口是：SparkContext
• 不同方式的编程，SparkContext略有差异：
• 在SparkCore编程中：
• java版本的SparkContext：JavaSparkContext
• scala版本的SparkContext:SparkContext
• 在SparkSQL编程中：

SQLContext/HiveContext
或者现在比较流行的方式：SparkSession

• 在SparkStreaming中：

StreamingContext

• 而SparkContext的构建，需要依赖SparkConf
  过程如下：

1、构建编程入口SparkContext
需要依赖SparkConf
指定相关参数信息
2、加载外部数据
生成操作的编程模型—RDD
3、基于RDD执行各种transformation操作
业务逻辑
4、作业的执行需要Action去触发
5、释放资源

一、Spark

1、java非λ表达式版（非集群模式）

?：
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.api.java.function.VoidFunction;
import scala.Tuple2;

import java.util.Arrays;
import java.util.Iterator;

public class _01WorldCount {
    public static void main(String[] args) {
        //1.指定spark程序的入口和相关配置
        SparkConf conf = new SparkConf();
        conf.setMaster("local[*]");
        conf.setAppName(_01WorldCount.class.getSimpleName());
        JavaSparkContext jsc = new JavaSparkContext(conf);

        //2.加载外部数据
        //这个file的地址因为我用的是mac所以这么写，如果是windows请换成对应的地址
        JavaRDD<String> lines = jsc.textFile("/Users/xxx/Desktop/words.txt");
        System.out.println("####partition num=>"+lines.getNumPartitions());

        //3.transformation
        JavaRDD<String> wordsRDD = lines.flatMap(new FlatMapFunction<String, String>() {

            @Override
            public Iterator<String> call(String line) throws Exception {
                return Arrays.asList(line.split("\\s+")).iterator();
            }
        });
        JavaPairRDD<String,Integer> pairsRDD = wordsRDD.mapToPair(new PairFunction<String, String, Integer>() {
            @Override
            public Tuple2<String, Integer> call(String word) throws Exception {
                return new Tuple2<String, Integer>(word,1);
            }
        });

        JavaPairRDD<String, Integer> retRDD = pairsRDD.reduceByKey(new Function2<Integer, Integer, Integer>() {
            @Override
            public Integer call(Integer v1, Integer v2) throws Exception {
                return v1+v2;
            }
        });

        //4.action触发作业
        retRDD.foreach(new VoidFunction<Tuple2<String, Integer>>() {
            @Override
            public void call(Tuple2<String, Integer> t) throws Exception {
                System.out.println(t._1+"---->"+t._2);
            }
        });

        //5.释放资源
        jsc.stop();
    }
}

运行结果：

2、javaλ表达式版（jdkv1.8+版本支持，非集群模式）

?：
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import scala.Tuple2;

import java.util.Arrays;


public class _02WordCount {
    public static void main(String[] args) {
        SparkConf conf = new SparkConf();
        conf.setMaster("local[*]");
        conf.setAppName(_02WordCount.class.getSimpleName());

        JavaSparkContext jsc = new JavaSparkContext(conf);
        //step2 加载外部数据
        JavaRDD<String> lines = jsc.textFile("/Users/xxx/Desktop/words.txt");
        JavaRDD<String> words = lines.flatMap(line -> Arrays.asList(line.split("\\s+")).iterator());
        JavaPairRDD<String, Integer> pairs = words.mapToPair(word -> new Tuple2<String, Integer>(word, 1));
        JavaPairRDD<String, Integer> ret = pairs.reduceByKey((v1, v2) -> v1 + v2);
        ret.foreach(t -> System.out.println(t._1+"---->"+t._2));
        
        jsc.stop();
    }
}

3、scala本地版

?：
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
  * scala版本的wordcount
  */
object _01WordCountScalaApp {
    def main(args: Array[String]): Unit = {
        val conf = new SparkConf()
	.setAppName(s"${_01WordCountScalaApp.getClass.getSimpleName}")
                .setMaster("local")
        val sc = new SparkContext(conf)
		//数据源数list
        val list = List(
            "cai zhang ting ting",
            "suo zhao zhao cao yu",
            "li guo long san yu"
        )
        val listRDD:RDD[String] = sc.parallelize(list)
        val retRDD:RDD[(String, Int)] = listRDD.flatMap(_.split("\\s+")).map((_, 1)).reduceByKey(_+_)
        retRDD.foreach(println)
        sc.stop()
    }
}

4、scala对hdfs上文件进行wordCount

（也可以提交到集群上执行！）
``记得需要将hdfs的配置文件core-site.xml和hdfs-site.xml添加到classpath下面，要不运行就会报java.net.UnknownHostException的错！```

?：
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
/**
  * 加载外部hdfs中的文件，进行计算
  * java.net.UnknownHostException: ns1
  *      将core-site.xml和hdfs-site.xml添加到classpath下面即可
  * 此时再去读取本地文件，便会报文件路径找不到，因为有了这两文件的原因，自动将传入的路径解析为hdfs的路，后续的本地文件格式写法如下：
  * windows是file:///E:/data/hello.txt
  * mac是file:///data/hello.txt
  */
object _02WordCountApp {
    def main(args: Array[String]): Unit = {
        val conf = new SparkConf()
                .setAppName(s"${_02WordCountApp.getClass.getSimpleName}")
                .setMaster("local")
        val sc = new SparkContext(conf)
//这里的ns1是hdfs集群的命名空间（我的hadoop安装的是HA高可用的）
        val listRDD:RDD[String] = sc.textFile("hdfs://ns1/data/spark/hello")

        val retRDD:RDD[(String, Int)] = listRDD.flatMap(_.split("\\s+")).map((_, 1)).reduceByKey(_+_)
        retRDD.foreach(println)

        sc.stop()
    }
}

5、使用spark算子groupByKey、reduceByKey、countByKey实现Wordcount

?：
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkContext, SparkConf}

object Wordcount{
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
      .setAppName("Wordcount")
      .setMaster("local[1]")
    val sc = new SparkContext(conf)
    val list = List(
      "hello you hello me",
      "hello you shit me",
      "hello you oh my",
      "me you hello me"
    )
    val listRDD = sc.parallelize(list)

    val pairsRDD: RDD[(String, Int)] = listRDD.flatMap(_.split("\\s+")).map((_, 1))

    val retRDD: RDD[(String, Int)] = pairsRDD.reduceByKey(_ + _)
    //通过每个key出现的次数
    val k2counts = pairsRDD.countByKey()
    for ((k, v) <- k2counts) {
      println(s"${k}---$v")
    }
    //使用groupByKey、reduceByKey、countByKey   --->wordcount
    println("-----------------------------------")
    val gbks = pairsRDD.groupByKey()
    //        gbks.foreach(println)
    gbks.map { case (key, values) => (key, values.size) }.foreach(println)
  }
}

二、MapReduce

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

/*
 * 封装main函数
 * 1）组装maper  reducer
 * 2）提交执行
 */
public class Driver {

	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		/*
		 * 
		 * 在mapreduce中将一个maper-reducer封装的一个任务交job
		 * 对应的类Job
		 */
		//加载hadoop配置文件
		Configuration conf= new Configuration();
		
		//启动一个job,封装mapper和reducer
		Job job = Job.getInstance(conf);
		job.setJarByClass(Driver.class);
		
		//设置这个job对应的mapper和reducer
		job.setMapperClass(WordCountMapper.class);
		job.setReducerClass(WordCountReducer.class);
		
		//设置map输出k  v类型
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(IntWritable.class);
		//设置reduce输出k v类型
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(IntWritable.class);
		//指定输入，需要统计的路径，mapreduce
		FileInputFormat.addInputPath(job, new Path(args[0]));
		//指定输出，统计结果路径
		FileOutputFormat.setOutputPath(job, new Path(args[1]));
		
		//提交job，job.submit()没有日志信息不建议使用
		//job.waitForCompletion()底层也调用了submite
		job.waitForCompletion(true);
	}
}
-------------------------------------------------
import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
public class WordCountMapper extends Mapper<LongWritable, Text, Text, IntWritable>{
	@Override
	protected void map(LongWritable key, 
			Text value, 
			Context context)
			throws IOException, InterruptedException {
		/*
		 * 逻辑
		 * 每次到map函数中，都是拿到一行内容
		 * 1）获取行内容
		 * 2）切分每个单词
		 * 3）发送给reduce   key：单词  value：1
		 * Text, IntWritable
		 */
		//1）获取行内容
		String line = value.toString();
		//2）切分每个单词
		String[] words = line.split("\t");
		for (String w : words) {
			//将java中的String转换成hadoop内容
			Text mk = new Text(w);
			IntWritable mv  = new IntWritable(1);
			context.write(mk, mv);
		}
	}
}
-------------------------------------------------
import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class WordCountReducer extends Reducer<Text, IntWritable, Text, IntWritable>{
	@Override
	protected void reduce(Text key, Iterable<IntWritable> values,
			Reducer<Text, IntWritable, Text, IntWritable>.Context context) throws IOException, InterruptedException {
		int sum = 0;
		for (IntWritable w : values) {
			//hadoop中的数值类型（int double...）---java数值   get
			sum += w.get();
		}
		context.write(key, new IntWritable(sum));
	}
}

三、hive

在hive中使用hql实现Wordcount：

select
tmp.word,
count(1) count 
from (select explode(split(line, '\\s+')) word from t_wc) tmp
group by tmp.word;