【Hadoop离线基础总结】MapReduce倒排索引建立

MapReduce倒排索引建立

---

求某些单词在文章中出现多少次

• 有三个文档的内容，求hello,tom,jerry三个单词在其中各出现多少次

hello tom

hello jerry

hello tom

hello jerry

hello jerry

tom jerry

hello jerry

hello tom

• java代码实现

定义一个Mapper类

package cn.itcast.demo2;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.InputSplit;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.lib.input.FileSplit;

import java.io.IOException;

public class IndexMapper extends Mapper<LongWritable, Text, Text, IntWritable> {

    @Override

    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

        //获取文件切片,强转!强转!

        FileSplit fileSplit = (FileSplit) context.getInputSplit();

        //获取文档名字

        String name = fileSplit.getPath().getName();

        //对v1进行切割

        String[] split = value.toString().split(" ");

        for (String s : split) {

            context.write(new Text(s + "-" + name), new IntWritable(1));

        }

    }

}

定义一个reducer类

package cn.itcast.demo2;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class IndexReducer extends Reducer<Text, IntWritable, Text, IntWritable> {

    @Override

    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {

        int i = 0;

        for (IntWritable value : values) {

            //获取单词出现的次数

            i += value.get();

        }

        context.write(key, new IntWritable(i));

    }

}

程序main函数入口

package cn.itcast.demo2;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.conf.Configured;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

import org.apache.hadoop.util.Tool;

import org.apache.hadoop.util.ToolRunner;

public class IndexMain extends Configured implements Tool {

    @Override

    public int run(String[] args) throws Exception {

        //获取job对象

        Job job = Job.getInstance(super.getConf(), "getIndex");

        //输入数据，设置输入路径

        job.setInputFormatClass(TextInputFormat.class);

        TextInputFormat.setInputPaths(job, new Path("file:////Volumes/赵壮备份/大数据离线课程资料/5.大数据离线第五天/倒排索引/input"));

        //自定义map逻辑

        job.setMapperClass(IndexMapper.class);

        //设置k2,v2输出类型

        job.setMapOutputKeyClass(Text.class);

        job.setMapOutputValueClass(IntWritable.class);

        //自定义reduce逻辑

        job.setReducerClass(IndexReducer.class);

        //设置k3,v3输出类型

        job.setOutputKeyClass(Text.class);

        job.setOutputValueClass(IntWritable.class);

        //输出数据，设置输出路径

        job.setOutputFormatClass(TextOutputFormat.class);

        TextOutputFormat.setOutputPath(job, new Path("file:////Volumes/赵壮备份/大数据离线课程资料/5.大数据离线第五天/倒排索引/output"));

        //提交任务到集群

        boolean b = job.waitForCompletion(true);

        return b ? 0 : 1;

    }

    public static void main(String[] args) throws Exception {

        int run = ToolRunner.run(new Configuration(), new IndexMain(), args);

        System.exit(run);

    }

}

输出结果

hello-a.txt	3

hello-b.txt	2

hello-c.txt	2

jerry-a.txt	1

jerry-b.txt	3

jerry-c.txt	1

tom-a.txt	2

tom-b.txt	1

tom-c.txt	1