java - hadoop作业失败，出现奇怪的异常

错误信息:

java.io.IOException: Type mismatch in value from map: expected org.apache.hadoop.io.Text, recieved org.apache.hadoop.io.LongWritable
        at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.collect(MapTask.java:918)
        at org.apache.hadoop.mapred.MapTask$OldOutputCollector.collect(MapTask.java:512)
        at main.java.countdestinfo.myTask$Map.map(myTask.java:66)
        at main.java.countdestinfo.myTask$Map.map(myTask.java:45)
        at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:50)
        at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:374)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:324)
        at org.apache.hadoop.mapred.Child.main(Child.java:165)

我的密码(此处仅 map 密码):

 44 public class myTask{
 45     public static class Map extends MapReduceBase implements Mapper<BytesWritable, BytesWritable, Text, LongWritable>{
 47         public void map(BytesWritable key, BytesWritable value,
 48                 OutputCollector<Text, LongWritable> output, Reporter reporter) throws IOException{
 66             output.collect(new Text("xxxxxxxxxxxxxxxxx" + "xxx"), new LongWritable(1));
 88         }
 89     }

此处省略的所有行均为注释。

我通过了Text，但声称收到了LongWritable ..

header :

  3 import java.util.ArrayList;
  4 import java.util.HashMap;
  5 import java.util.List;
  6 import java.util.*;
  7 import java.lang.*;
  8 import java.io.IOException;
  9 import java.util.Iterator;
 10
 11 import org.apache.hadoop.conf.Configured;
 12 import org.apache.hadoop.fs.FileSystem;
 13 import org.apache.hadoop.fs.Path;
 14 import org.apache.hadoop.io.SequenceFile;
 15 import org.apache.hadoop.io.Text;
 16 import org.apache.hadoop.io.*;
 17 import org.apache.hadoop.mapred.JobClient;
 18 import org.apache.hadoop.mapred.JobConf;
 19 import org.apache.hadoop.mapred.MapReduceBase;
 20 import org.apache.hadoop.mapred.Mapper;
 21 import org.apache.hadoop.mapred.OutputCollector;
 22 import org.apache.hadoop.mapred.Reducer;
 23 import org.apache.hadoop.mapred.Reporter;
 24 import org.apache.hadoop.mapred.SequenceFileInputFormat;
 25 import org.apache.hadoop.util.GenericOptionsParser;
 26 import org.apache.hadoop.util.Tool;
 27 import org.apache.hadoop.mapred.FileInputFormat;
 28 import org.apache.hadoop.mapred.FileOutputFormat;
 29 import org.apache.hadoop.mapred.TextInputFormat;
 30 import org.apache.hadoop.mapred.TextOutputFormat;
 31
 32
 33 import java.io.InputStreamReader;
 34 import java.util.HashSet;
 35 import java.util.Iterator;
 37 import org.apache.hadoop.io.LongWritable;
 38 import org.apache.hadoop.io.BytesWritable;
 39
 40 import com.google.protobuf.InvalidProtocolBufferException;
 41 import display.algo.logs.proto.MiddataMessage;
 42 import java.util.Arrays;

减少和主要功能:

 90     public static class Reduce extends MapReduceBase implements Reducer<Text, LongWritable, Text, Text>{
 91         public void reduce(Text key, Iterator<Text>values,
 92                 OutputCollector<Text, LongWritable> output, Reporter reporter) throws IOException{
 93             long sum = 0;
 94             while(values.hasNext()){
 95                 sum += 1;
 96             }
 97             output.collect(key,new Text(String.valueOf(sum)));
 98         }
 99     }
100
101     public static void main(String[] args) throws Exception {
102         JobConf conf = new JobConf(myTask.class);
103         conf.setJobName("myTask");
104
105         conf.setOutputKeyClass(Text.class);
106         conf.setOutputValueClass(Text.class);
107
108         conf.setMapperClass(Map.class);
109         conf.setCombinerClass(Reduce.class);
110         conf.setReducerClass(Reduce.class);
111
112         conf.setInputFormat(SequenceFileInputFormat.class);
113         conf.setOutputFormat(TextOutputFormat.class);
114         conf.setStrings("mapred.reduce.tasks", "500");
115
120         FileInputFormat.addInputPath(conf, new Path(args[0]));
121         FileOutputFormat.setOutputPath(conf, new Path(args[1]));
122         JobClient.runJob(conf);
123
124     }

最佳答案

reduce的输入值应为LongWritable

public void reduce(Text key, Iterator<Text>values,
 OutputCollector<Text, LongWritable> output, Reporter reporter)

应该:

public void reduce(Text key, Iterator<LongWritable>values,
 OutputCollector<Text, Text> output, Reporter reporter)