Hadoop——MapReduce初级例子（java）

最新推荐文章于 2025-01-10 18:14:51 发布

原创最新推荐文章于 2025-01-10 18:14:51 发布 · 836 阅读

0 ·

本内容遵循CC 4.0 BY-SA版权协议

本文介绍了一种基于Hadoop实现的数据去重方法，并提供了具体的Java代码实现细节。该方法利用MapReduce框架处理重复数据记录，适用于大规模数据集。

一，数据去重

package dedup_test;

import java.io.IOException;    
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;   
import org.apache.hadoop.io.*;  
import org.apache.hadoop.mapreduce.Job;  
import org.apache.hadoop.mapreduce.Mapper;  
import org.apache.hadoop.mapreduce.Reducer;  
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;  
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;  
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.util.Tool;

public class dedup_test extends Configured implements Tool {

	public static class Map extends Mapper<Object, Text, Text, Text> {

		private static Text line = new Text();

		public void map(Object key, Text value, Context context)
				throws IOException, InterruptedException {

			line = value;
			context.write(line, new Text(""));
		}
	}

	public static class Reduce extends Reducer<Text, Text, Text, Text> {

		public void reduce(Text key, Iterable<Text> values, Context context)
				throws IOException, InterruptedException {

			context.write(key, new Text(""));
		}
	}

	public int run(String[] args) throws Exception {

		Configuration conf = getConf();

		String[] ioArgs = new String[] { "dedup_in", "dedup_out" };
		String[] otherArgs = new GenericOptionsParser(conf, ioArgs).getRemainingArgs();

		if (otherArgs.length != 2) {
			System.err.println("Usage: Data Deduplication <in> <out>");
			System.exit(2);
		}

		Job job = Job.getInstance(conf);
		job.setJarByClass(dedup_test.class);

		job.setMapperClass(Map.class);
		job.setCombinerClass(Reduce.class);
		job.setReducerClass(Reduce.class);

		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(Text.class);

		FileInputFormat.addInputPath(job, new Path(args[0]));
		FileOutputFormat.setOutputPath(job, new Path(args[1]));
		System.exit(job.waitForCompletion(true) ? 0 : 1);

		return job.isSuccessful() ? 0 : 1;
	}

	public static void main(String[] args) throws Exception {
		int res = ToolRunner.run(new Configuration(), new dedup_test(), args);
		System.exit(res);
	}
}

二，数据排序

三，算出多个学生的平均成绩