object serialization for Map Reduce - hadoop

Question

Hello Friends,
I am trying to serialize object, which can be pass to reducer from mapper as out value. And in this program I am getting below exception
java.lang.RuntimeException: java.lang.NoSuchMethodException: com.test.objectpass.SerObj.<init>()
at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:115)
at org.apache.hadoop.io.serializer.WritableSerialization$WritableDeserializer.deserialize(WritableSerialization.java:62)
at org.apache.hadoop.io.serializer.WritableSerialization$WritableDeserializer.deserialize(WritableSerialization.java:40)
at org.apache.hadoop.mapreduce.ReduceContext.nextKeyValue(ReduceContext.java:116)
at org.apache.hadoop.mapreduce.ReduceContext.nextKey(ReduceContext.java:92)
at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:175)
at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:649)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:417)
at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:260)
Caused by: java.lang.NoSuchMethodException: com.test.objectpass.SerObj.<init>()
at java.lang.Class.getConstructor0(Class.java:2715)
at java.lang.Class.getDeclaredConstructor(Class.java:1987)
at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:109)
I am copying code for object class which needs to be serialize -
package com.test.objectpass;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.io.WritableComparable;
public class SerObj implements WritableComparable<Object> {
private String name = null;
private String surname = null;
private Long number = null;
public SerObj(String name, String surname, Long number) {
super();
setName(name);
setNumber(number);
setSurname(surname);
}
#Override
public final String toString() {
final StringBuilder string = new StringBuilder();
string.append("SerObj [name=").append(name).append(", surname=")
.append(surname).append(", number=").append(number).append("]");
return string.toString();
}
public final String getName() { return name; }
public final void setName(String name) { this.name = name; }
public final String getSurname() { return surname; }
public final void setSurname(String surname) { this.surname = surname; }
public final Long getNumber() { return number; }
public final void setNumber(Long number) { this.number = number; }
#Override
public void readFields(DataInput in) throws IOException {
name =in.readLine();
surname = in.readLine();
number = in.readLong();
}
#Override
public void write(DataOutput out) throws IOException {
out.writeBytes(name);
out.writeBytes(surname);
out.writeLong(number);
}
#Override
public boolean equals(Object o) {
if (!(o instanceof SerObj))
return false;
SerObj other = (SerObj) o;
return this.number == other.number;
}
#Override
public int compareTo(Object o) {
long thisValue = this.number;
long thatValue = ((SerObj)o).number;
return (thisValue<thatValue ? -1 : (thisValue==thatValue ? 0 : 1));
}
}
Below code is for map reduce drive where I am submitting job -
package com.test.objectpass;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
public class ObjectSerialization {
public static class MyMapper extends Mapper<LongWritable, Text, Text, SerObj> {
#Override
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String[] values = value.toString().split(" ");
SerObj obj = new SerObj(values[0], values[1], Long.parseLong(values[2]));
context.write(new Text(values[0]), obj);
}
}
public static class MyReducer extends Reducer<Text, SerObj, NullWritable, NullWritable> {
#Override
public void reduce(Text key, Iterable<SerObj> values, Context context) throws IOException, InterruptedException {
for (SerObj valueObj : values) {
System.out.println(valueObj);
}
}
}
public static void main(String[] args) throws Exception {
final Configuration conf = new Configuration();
Job job = new Job(conf, "TEST");
job.setJarByClass(ObjectSerialization.class);
job.setMapperClass(MyMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(SerObj.class);
job.setReducerClass(MyReducer.class);
job.setOutputFormatClass(NullOutputFormat.class);
TextInputFormat.addInputPath(job, new Path("/home/pankaj/test"));
job.waitForCompletion(true);
System.out.println("Done.");
}
}
Please help me to solve this problem.
Thanks in advance.

Answer 1

The error message should be clear enough; there is no 0-arg constructor in your SerObj class. Nearly every serialization framework in existence requires that your beans have no-arg constructors available so that the framework can instantiate something via reflection before having read all of its data off the wire, and Writable serialization is no different.