Null pointer exception in hadoop reducer - hadoop

Question

I am facing the NullPointerException with the below code. It would be great if some one can review and help me with the program.
The mapper is running fine but, I get an NPE, when I am try to split the value at the iterator. Please help me figure out my mistake. I have attached the mapper out below.
Toppermain.java
package TopperPackage;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
public class TopperMain {
//hadoop jar worcount.jar ars[0] args[1]
public static void main(String[] args) throws Exception {
Job myhadoopJob = new Job();
myhadoopJob.setJarByClass(TopperMain.class);
myhadoopJob.setJobName("Finding topper based on subject");
FileInputFormat.addInputPath(myhadoopJob, new Path(args[0]));
FileOutputFormat.setOutputPath(myhadoopJob, new Path(args[1]));
myhadoopJob.setInputFormatClass(TextInputFormat.class);
myhadoopJob.setOutputFormatClass(TextOutputFormat.class);
myhadoopJob.setMapperClass(TopperMapper.class);
myhadoopJob.setReducerClass(TopperReduce.class);
myhadoopJob.setMapOutputKeyClass(Text.class);
myhadoopJob.setMapOutputValueClass(Text.class);
myhadoopJob.setOutputKeyClass(Text.class);
myhadoopJob.setOutputValueClass(Text.class);
System.exit(myhadoopJob.waitForCompletion(true) ? 0 : 1);
}
}
TopperMapper.java
package TopperPackage;
import java.io.IOException;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
/*Surender,87,60,50,50,80
Raj,80,70,80,85,60
Anten,81,60,50,70,100
Dinesh,60,90,80,80,70
Priya,80,85,91,60,75
*/
public class TopperMapper extends Mapper<LongWritable, Text, Text, Text>
{
String temp,temp2;
protected void map(LongWritable key, Text value,Context context)
throws IOException, InterruptedException {
String record = value.toString();
String[] parts = record.split(",");
temp=parts[0];
temp2=temp+ "\t" + parts[1];
context.write(new Text("Tamil"),new Text(temp2));
temp2=temp+ "\t" + parts[2];
context.write(new Text("English"),new Text(temp2));
temp2=temp+ "\t" + parts[3];
context.write(new Text("Maths"),new Text(temp2));
temp2=temp+ "\t" + parts[4];
context.write(new Text("Science"),new Text(temp2));
temp2=temp+ "\t" + parts[5];
context.write(new Text("SocialScrience"),new Text(temp2));
}
}
TopperReduce.java
package TopperPackage;
import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
public class TopperReduce extends Reducer<Text, Text, Text, Text> {
int temp;
private String[] names;
private int[] marks;
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
String top = "";
int count =0,topmark;
marks = null;
String befsplit;
String[] parts=null;
names = null;
for (Text t : values)
{
befsplit= t.toString();
parts = befsplit.split("\t");
names[count]=parts[0];
marks[count]=Integer.parseInt(parts[1]);
count = count+1;
}
topmark=calcTopper(marks);
top=names[topmark]+ "\t"+marks[topmark] ;
context.write(new Text(key), new Text(top));
}
public int calcTopper(int[] marks)
{
int count=marks.length;
temp=((marks[1]));
int i=0;
for (i=1;i<=(count-2);i++)
{
if(temp < marks[i+1])
{
temp = marks[i+1];
}
}
return i;
}
}
the error is
cloudera#cloudera-vm:~/Jarfiles$ hadoop jar TopperMain.jar /user/cloudera/inputfiles/topper/topperinput.txt /user/cloudera/outputfiles/topper/
14/08/24 23:17:07 WARN mapred.JobClient: Use GenericOptionsParser for parsing the arguments. Applications should implement Tool for the same.
14/08/24 23:17:08 INFO input.FileInputFormat: Total input paths to process : 1
14/08/24 23:17:09 INFO mapred.JobClient: Running job: job_201408241907_0012
14/08/24 23:17:10 INFO mapred.JobClient: map 0% reduce 0%
14/08/24 23:17:49 INFO mapred.JobClient: map 100% reduce 0%
14/08/24 23:18:03 INFO mapred.JobClient: Task Id : attempt_201408241907_0012_r_000000_0, Status : FAILED
java.lang.NullPointerException
at TopperPackage.TopperReduce.reduce(TopperReduce.java:25)
at TopperPackage.TopperReduce.reduce(TopperReduce.java:1)
at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:176)
at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:571)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:413)
at org.apache.hadoop.mapred.Child$4.run(Child.java:268)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1115)
at org.apache.hadoop.mapred.Child.main(Child.java:262)
attempt_201408241907_0012_r_000000_0: log4j:WARN No appenders could be found for logger (org.apache.hadoop.hdfs.DFSClient).
attempt_201408241907_0012_r_000000_0: log4j:WARN Please initialize the log4j system properly.
14/08/24 23:18:22 INFO mapred.JobClient: Task Id : attempt_201408241907_0012_r_000000_1, Status : FAILED
java.lang.NullPointerException
at TopperPackage.TopperReduce.reduce(TopperReduce.java:25)
at TopperPackage.TopperReduce.reduce(TopperReduce.java:1)
at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:176)
at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:571)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:413)
at org.apache.hadoop.mapred.Child$4.run(Child.java:268)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1115)
at org.apache.hadoop.mapred.Child.main(Child.java:262)
attempt_201408241907_0012_r_000000_1: log4j:WARN No appenders could be found for logger (org.apache.hadoop.hdfs.DFSClient).
attempt_201408241907_0012_r_000000_1: log4j:WARN Please initialize the log4j system properly.
I am getting the expected output from mapper but reducer is throwing error when splitting the output and storing in a variable.
The mapper output is
Tamil Surender 87
English Surender 60
Maths Surender 50
Science Surender 50
SocialScrience Surender 80
Tamil Raj 80
English Raj 70
Maths Raj 80
Science Raj 85
SocialScrience Raj 60
Tamil Anten 81
English Anten 60
Maths Anten 50
Science Anten 70
SocialScrience Anten 100
Tamil Dinesh 60
English Dinesh 90
Maths Dinesh 80
Science Dinesh 80
SocialScrience Dinesh 70
Tamil Priya 80
English Priya 85
Maths Priya 91
Science Priya 60
SocialScrience Priya 75
Any advice to point out my mistake is appreciated.

Answer 1

The error is due to you are initializing marks and names arrays to null and not initialize them properly. Please use the below reducer class.
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
public class TopperReduce extends Reducer<Text, Text, Text, Text> {
int temp;
private String[] names = new String[10];
private int[] marks = new int[10];
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
String top = "";
int count = 0, topmark;
String befsplit;
String[] parts = null;
for (Text t : values) {
befsplit = t.toString();
parts = befsplit.split("\t");
names[count] = parts[0];
marks[count] = Integer.parseInt(parts[1]);
count++;
}
topmark = calcTopper(marks);
top = names[topmark] + "\t" + marks[topmark];
context.write(new Text(key), new Text(top));
}
public int calcTopper(int[] marks) {
int count = marks.length;
int i = 0;
int highestMArk = 0;
int mark = 0;
int highestMarkIndex = 0;
for (; i < count; i++) {
mark = marks[i];
if (mark > highestMArk) {
highestMarkIndex = i;
}
}
return highestMarkIndex;
}
}

Answer 2

You are referring to a null array variable parts so you are getting this error,
change your code as i mentioned below it could work
public class TopperReduce extends Reducer<Text, Text, Text, Text> {
int temp;
private String[] names=new String[20];
private int[] marks= new int[20];
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
String top = "";
int count =0,topmark;
for (Text t : values)
{
String befsplit= t.toString();
String[] parts = befsplit.split("\t");
names[count]=parts[0];
marks[count]=Integer.parseInt(parts[1]);
count = count+1;
}
topmark=calcTopper(marks);
top=names[topmark]+ "\t"+marks[topmark] ;
context.write(new Text(key), new Text(top));
}