MapReduce编程之wordcount

实践

MapReduce编程之wordcount

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

/**

 * 使用MapReduce开发WordCount的应用程序

 */

public class WordCountApp {

    /**

     * Map：读取输入的文件

     */

    public static class MyMapper extends Mapper<LongWritable,Text,Text,LongWritable>{

        LongWritable one = new LongWritable(1);

        @Override

        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

            // 接收到的每一行数据

            String line = value.toString();

            //按照指定分隔符进行拆分

            String[] words = line.split(" ");

            for(String word : words){

                // 通过上下文把map的处理结果输出

                context.write(new Text(word),one);

            }

        }

    }

    /**

     * 归并操作

     */

    public static class MyReduce extends Reducer<Text,LongWritable,Text,LongWritable>{

        @Override

        protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {

            long sum = 0;

            for(LongWritable value : values){

                //求key出现的次数和

                sum += value.get();

            }

            context.write(key, new LongWritable(sum));

        }

    }

    /**

     * 定义Driver：封装lMapReduce作业的所有信息

     * @param args

     */

    public static void main(String[] args) throws Exception{

        //创建configuration

        Configuration configuration = new Configuration();

        //准备清理已存在的输出目录

        Path outputPath = new Path(args[1]);

        FileSystem fileSystem = FileSystem.get(configuration);

        if(fileSystem.exists(outputPath)){

            fileSystem.delete(outputPath,true);

            System.out.println("out file exists,but is has deleted!");

        }

        //创建job

        Job job = Job.getInstance(configuration,"WordCount");

        //设置job的处理类

        job.setJarByClass(WordCountApp.class);

        //设置作业处理的输入路径

        FileInputFormat.setInputPaths(job,new Path(args[0]));

        //设置map相关参数

        job.setMapperClass(MyMapper.class);

        job.setMapOutputKeyClass(Text.class);

        job.setMapOutputValueClass(LongWritable.class);

        //设置reduce相关参数

        job.setReducerClass(MyReduce.class);

        job.setOutputKeyClass(Text.class);

        job.setOutputValueClass(LongWritable.class);

        //设置作业处理的输出路径

        FileOutputFormat.setOutputPath(job , new Path(args[1]));

        System.exit(job.waitForCompletion(true) ? 0 : 1);

    }

}

运行

hadoop jar hadoop-train-1.0-SNAPSHOT.jar WordCountApp /hdfsapi/test/b.txt /hdfsapi/test/out

MapReduce编程之Combiner

• 本地reduce（map端reduce）

• 减少Map Tasks输出的数据量及数据网络传输量

• combiner案例开发

• 使用场景：求和、求次数

• 代码

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

/**

 * 使用MapReduce开发WordCount的应用程序

 */

public class CombinerApp {

    /**

     * Map：读取输入的文件

     */

    public static class MyMapper extends Mapper<LongWritable,Text,Text,LongWritable>{

        LongWritable one = new LongWritable(1);

        @Override

        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

            // 接收到的每一行数据

            String line = value.toString();

            //按照指定分隔符进行拆分

            String[] words = line.split(" ");

            for(String word : words){

                // 通过上下文把map的处理结果输出

                context.write(new Text(word),one);

            }

        }

    }

    /**

     * 归并操作

     */

    public static class MyReduce extends Reducer<Text,LongWritable,Text,LongWritable>{

        @Override

        protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {

            long sum = 0;

            for(LongWritable value : values){

                //求key出现的次数和

                sum += value.get();

            }

            context.write(key, new LongWritable(sum));

        }

    }

    /**

     * 定义Driver：封装lMapReduce作业的所有信息

     * @param args

     */

    public static void main(String[] args) throws Exception{

        //创建configuration

        Configuration configuration = new Configuration();

        //准备清理已存在的输出目录

        Path outputPath = new Path(args[1]);

        FileSystem fileSystem = FileSystem.get(configuration);

        if(fileSystem.exists(outputPath)){

            fileSystem.delete(outputPath,true);

            System.out.println("out file exists,but is has deleted!");

        }

        //创建job

        Job job = Job.getInstance(configuration,"WordCount");

        //设置job的处理类

        job.setJarByClass(CombinerApp.class);

        //设置作业处理的输入路径

        FileInputFormat.setInputPaths(job,new Path(args[0]));

        //设置map相关参数

        job.setMapperClass(MyMapper.class);

        job.setMapOutputKeyClass(Text.class);

        job.setMapOutputValueClass(LongWritable.class);

        //设置reduce相关参数

        job.setReducerClass(MyReduce.class);

        job.setOutputKeyClass(Text.class);

        job.setOutputValueClass(LongWritable.class);

        //通过job的设置combiner处理类，其实逻辑上和我们的reduce是一模一样的

        job.setCombinerClass(MyReduce.class);

        //设置作业处理的输出路径

        FileOutputFormat.setOutputPath(job , new Path(args[1]));

        System.exit(job.waitForCompletion(true) ? 0 : 1);

    }

}

• 执行命令

hadoop jar hadoop-train-1.0-SNAPSHOT.jar WordCountApp /hdfsapi/test/b.txt /hdfsapi/test/out

MapReduce编程之Partitioner

• partitioner决定MapTask输出的数据交由哪个ReduceTask处理

• 默认实现：分发的key的hash值对ReduceTask个数取模

• partitioner案例开发

• 代码

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Partitioner;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

/**

 * 使用MapReduce开发WordCount的应用程序

 */

public class PartitionerApp {

    /**

     * Map：读取输入的文件

     */

    public static class MyMapper extends Mapper<LongWritable,Text,Text,LongWritable>{

        LongWritable one = new LongWritable(1);

        @Override

        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

            // 接收到的每一行数据

            String line = value.toString();

            //按照指定分隔符进行拆分

            String[] words = line.split(" ");

            context.write(new Text(words[0]),new LongWritable(Long.parseLong(words[1])));

        }

    }

    /**

     * 归并操作

     */

    public static class MyReduce extends Reducer<Text,LongWritable,Text,LongWritable>{

        @Override

        protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {

            long sum = 0;

            for(LongWritable value : values){

                //求key出现的次数和

                sum += value.get();

            }

            context.write(key, new LongWritable(sum));

        }

    }

    public static class MyPartitioner extends Partitioner<Text,LongWritable>{

        @Override

        public int getPartition(Text key, LongWritable longWritable, int i) {

            if(key.toString().equals("xiaomi")){

                return 0;

            }

            if(key.toString().equals("huawei")){

                return 1;

            }

            if(key.toString().equals("iphone")){

                return 2;

            }

            return 3;

        }

    }

    /**

     * 定义Driver：封装lMapReduce作业的所有信息

     * @param args

     */

    public static void main(String[] args) throws Exception{

        //创建configuration

        Configuration configuration = new Configuration();

        //准备清理已存在的输出目录

        Path outputPath = new Path(args[1]);

        FileSystem fileSystem = FileSystem.get(configuration);

        if(fileSystem.exists(outputPath)){

            fileSystem.delete(outputPath,true);

            System.out.println("out file exists,but is has deleted!");

        }

        //创建job

        Job job = Job.getInstance(configuration,"WordCount");

        //设置job的处理类

        job.setJarByClass(PartitionerApp.class);

        //设置作业处理的输入路径

        FileInputFormat.setInputPaths(job,new Path(args[0]));

        //设置map相关参数

        job.setMapperClass(MyMapper.class);

        job.setMapOutputKeyClass(Text.class);

        job.setMapOutputValueClass(LongWritable.class);

        //设置reduce相关参数

        job.setReducerClass(MyReduce.class);

        job.setOutputKeyClass(Text.class);

        job.setOutputValueClass(LongWritable.class);

        //通过job的设置partition

        job.setPartitionerClass(MyPartitioner.class);

        //设置4个reduce，每个分区一个

        job.setNumReduceTasks(4);

        //设置作业处理的输出路径

        FileOutputFormat.setOutputPath(job , new Path(args[1]));

        System.exit(job.waitForCompletion(true) ? 0 : 1);

    }

}

• 执行命令

hadoop jar hadoop-train-1.0-SNAPSHOT.jar PartitionerApp /hdfsapi/test/partitioner /hdfsapi/test/outpartitioner