本文介绍了通过ArrayWritable迭代 - NoSuchMethodException的处理方法，对大家解决问题具有一定的参考价值，需要的朋友们下面随着小编来一起学习吧！
问题描述
我刚开始使用MapReduce，我遇到了一个我无法通过Google解答的奇怪的错误。我使用ArrayWritable编写基本程序，但是当我运行它时，在Reduce期间出现以下错误：
java。 lang.RuntimeException：
org.apache.hadoop.util.ReflectionUtils.newInstance中的java.lang.NoSuchMethodException：org.apache.hadoop.io.ArrayWritable。< init>（）
（ReflectionUtils.java ：115）
at org.apache.hadoop.io.serializer.WritableSerialization $ WritableDeserializer.deserialize（WritableSerialization.java:62）
at org.apache.hadoop.io.serializer.WritableSerialization $ WritableDeserializer.deserialize （WritableSerialization.java:40）
at org.apache.hadoop.mapred.Task $ ValuesIterator.readNextValue（Task.java:1276）
at org.apache.hadoop.mapred.Task $ ValuesIterator.next （Task.java:1214）
at org.apache.hadoop.mapred.ReduceTask $ ReduceValuesIterator.moveToNext（ReduceTask.java:250）
at org.apache.hadoop.mapred.ReduceTask $ ReduceValuesIterator.next （ReduceTask.java:246）PageRank
$ Reduce.reduce（未知源）PageRank
$ Re duce.reduce（Unknown Source）
at org.apache.hadoop.mapred.ReduceTask.runOldReducer（ReduceTask.java:522）
at org.apache.hadoop.mapred.ReduceTask.run（ReduceTask.java ：421）
at org.apache.hadoop.mapred.Child $ 4.run（Child.java:255）
我正在使用Hadoop 1.2.1。这里是我的代码：
import org.apache.hadoop.conf.Configuration;
导入org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred。*;
import org.apache.hadoop.io。*;
import org.apache.hadoop.mapred.join。*;
import java.io.IOException;
import java.util.Iterator;
public class TempClass {
public static class MapClass extends MapReduceBase
implements Mapper< LongWritable，Text，Text，ArrayWritable> {
public void map（LongWritable key，Text value，
OutputCollector< Text，ArrayWritable> output，
Reporter reporter）throws IOException {
String [] arr_str = new String [] {a，b，c};
for（int i = 0; i output.collect（new Text（my_key），new ArrayWritable（arr_str））;
$ b $ public static class Reduce extends MapReduceBase
implements Reducer< Text，ArrayWritable，Text，ArrayWritable> {
$ b $ public void reduce（Text key，Iterator< ArrayWritable> values，
OutputCollector< Text，ArrayWritable> output，
Reporter reporter）throws IOException {
ArrayWritable tmp;
while（values.hasNext（））{
tmp = values.next（）;
output.collect（key，tmp）;
$ b public static void main（String [] args）throws Exception {
Configuration conf = new Configuration（）;
JobConf job = new JobConf（conf，TempClass.class）;
job.setOutputKeyClass（Text.class）;
job.setOutputValueClass（ArrayWritable.class）;
job.setOutputFormat（TextOutputFormat.class）;
job.setInputFormat（TextInputFormat.class）;
job.setMapperClass（MapClass.class）;
job.setReducerClass（Reduce.class）;
FileInputFormat.setInputPaths（job，new Path（args [0]））;
FileOutputFormat.setOutputPath（job，new Path（args [1]））;
job.setJobName（TempClass）;
JobClient.runJob（job）;
$ b $如果我在下面的行注释（Reduce Class）： //while(values.hasNext（））{
// tmp = values.next（）;
output.collect（key，tmp）;
//}
一切都会好的。你有什么想法吗？
解决方案这是来自。通常， Writable 应该有一个没有参数的构造函数。
我只是修改了代码：
import java.io.IOException;
import java.util.Iterator;
导入org.apache.hadoop.conf.Configuration;
导入org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.ArrayWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapred.TextOutputFormat;
public class TempClass {
public static class TextArrayWritable extends ArrayWritable {
public TextArrayWritable（）{
super（Text.class）;
public TextArrayWritable（String [] strings）{
super（Text.class）;
Text [] texts = new Text [strings.length];
for（int i = 0; i< strings.length; i ++）{
texts [i] = new Text（strings [i]）;
}
set（texts）;
public static class MapClass extends MapReduceBase implements
Mapper< LongWritable，Text，Text，ArrayWritable> {
public void map（LongWritable key，Text value，
OutputCollector< Text，ArrayWritable> output，Reporter reporter）
throws IOException {
String [] arr_str = new String [] {
a，b，c};
for（int i = 0; i output.collect（new Text（my_key），new TextArrayWritable（
arr_str））;
public static class Reduce extends MapReduceBase implements
Reducer< Text，TextArrayWritable，Text，TextArrayWritable> {
$ b $ public void reduce（Text key，Iterator< TextArrayWritable> values，
OutputCollector< Text，TextArrayWritable>输出，
Reporter记者）抛出IOException {
TextArrayWritable tmp;
while（values.hasNext（））{
tmp = values.next（）;
output.collect（key，tmp）;
$ b public static void main（String [] args）throws Exception {
Configuration conf = new Configuration（）;
JobConf job = new JobConf（conf，TempClass.class）;
job.setOutputKeyClass（Text.class）;
job.setOutputValueClass（TextArrayWritable.class）;
job.setOutputFormat（TextOutputFormat.class）;
job.setInputFormat（TextInputFormat.class）;
job.setMapperClass（MapClass.class）;
job.setReducerClass（Reduce.class）;
FileInputFormat.setInputPaths（job，new Path（args [0]））;
FileOutputFormat.setOutputPath（job，new Path（args [1]））;
job.setJobName（TempClass）;
JobClient.runJob（job）;
}
}
I just started working with MapReduce, and I'm running into a weird bug that I haven't been able to answer through Google. I'm making a basic program using ArrayWritable, but when I run it, I get the following error during Reduce:
java.lang.RuntimeException:
java.lang.NoSuchMethodException:org.apache.hadoop.io.ArrayWritable.<init>()
at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:115)
at org.apache.hadoop.io.serializer.WritableSerialization$WritableDeserializer.deserialize(WritableSerialization.java:62)
at org.apache.hadoop.io.serializer.WritableSerialization$WritableDeserializer.deserialize(WritableSerialization.java:40)
at org.apache.hadoop.mapred.Task$ValuesIterator.readNextValue(Task.java:1276)
at org.apache.hadoop.mapred.Task$ValuesIterator.next(Task.java:1214)
at org.apache.hadoop.mapred.ReduceTask$ReduceValuesIterator.moveToNext(ReduceTask.java:250)
at org.apache.hadoop.mapred.ReduceTask$ReduceValuesIterator.next(ReduceTask.java:246)
at PageRank$Reduce.reduce(Unknown Source)
at PageRank$Reduce.reduce(Unknown Source)
at org.apache.hadoop.mapred.ReduceTask.runOldReducer(ReduceTask.java:522)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:421)
at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
I'm using Hadoop 1.2.1. Here's my code:
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.join.*;
import java.io.IOException;
import java.util.Iterator;
public class TempClass {
public static class MapClass extends MapReduceBase
implements Mapper<LongWritable, Text, Text, ArrayWritable> {
public void map(LongWritable key, Text value,
OutputCollector<Text, ArrayWritable> output,
Reporter reporter) throws IOException {
String[] arr_str = new String[]{"a","b","c"};
for(int i=0; i<3; i++)
output.collect(new Text("my_key"), new ArrayWritable(arr_str));
}
}
public static class Reduce extends MapReduceBase
implements Reducer<Text, ArrayWritable, Text, ArrayWritable> {
public void reduce(Text key, Iterator<ArrayWritable> values,
OutputCollector<Text, ArrayWritable> output,
Reporter reporter) throws IOException {
ArrayWritable tmp;
while(values.hasNext()){
tmp = values.next();
output.collect(key, tmp);
}
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
JobConf job = new JobConf(conf, TempClass.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(ArrayWritable.class);
job.setOutputFormat(TextOutputFormat.class);
job.setInputFormat(TextInputFormat.class);
job.setMapperClass(MapClass.class);
job.setReducerClass(Reduce.class);
FileInputFormat.setInputPaths( job, new Path( args[0] ) );
FileOutputFormat.setOutputPath( job, new Path( args[1] ) );
job.setJobName( "TempClass" );
JobClient.runJob(job);
}
}
If I comment below lines (Reduce Class):
//while(values.hasNext()){
// tmp = values.next();
output.collect(key, tmp);
//}
everything will be ok. Do you have any ideas?
解决方案 Here is from the docs of ArrayWritable. Generally, Writable should have an constructor with no parameter.
I just modified your code to:
import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.ArrayWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapred.TextOutputFormat;
public class TempClass {
public static class TextArrayWritable extends ArrayWritable {
public TextArrayWritable() {
super(Text.class);
}
public TextArrayWritable(String[] strings) {
super(Text.class);
Text[] texts = new Text[strings.length];
for (int i = 0; i < strings.length; i++) {
texts[i] = new Text(strings[i]);
}
set(texts);
}
}
public static class MapClass extends MapReduceBase implements
Mapper<LongWritable, Text, Text, ArrayWritable> {
public void map(LongWritable key, Text value,
OutputCollector<Text, ArrayWritable> output, Reporter reporter)
throws IOException {
String[] arr_str = new String[] {
"a", "b", "c" };
for (int i = 0; i < 3; i++)
output.collect(new Text("my_key"), new TextArrayWritable(
arr_str));
}
}
public static class Reduce extends MapReduceBase implements
Reducer<Text, TextArrayWritable, Text, TextArrayWritable> {
public void reduce(Text key, Iterator<TextArrayWritable> values,
OutputCollector<Text, TextArrayWritable> output,
Reporter reporter) throws IOException {
TextArrayWritable tmp;
while (values.hasNext()) {
tmp = values.next();
output.collect(key, tmp);
}
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
JobConf job = new JobConf(conf, TempClass.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(TextArrayWritable.class);
job.setOutputFormat(TextOutputFormat.class);
job.setInputFormat(TextInputFormat.class);
job.setMapperClass(MapClass.class);
job.setReducerClass(Reduce.class);
FileInputFormat.setInputPaths(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
job.setJobName("TempClass");
JobClient.runJob(job);
}
}
这篇关于通过ArrayWritable迭代 - NoSuchMethodException的文章就介绍到这了，希望我们推荐的答案对大家有所帮助，也希望大家多多支持！