Mapper not invoked while using multipleInputFormat - hadoop

Question

I have a driver class which is using MultipleInputFormat class to invoke different mappers at runtime.
However when I use the MultipleInputs.addInputPath(job, fStatus.getPath(), TextInputFormat.class,CreatePureDeltaMapperOne.class) in the first for loop, my first mapper(CreatePureDeltaMapperOne) is not getting invoked. When I comment the block of code which invokes the multiple input format from the first for loop, and call it from outside, the mapper class is invoked. Please help me find the issue.
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URISyntaxException;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.MultipleInputs;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
/***
* Creates the pure delta file by matching the history records present in HDFS
* #author Debajit
*
*/
public class CreatePureDeltaDriver {
/**
* #param args
* #throws URISyntaxException
*/
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException, URISyntaxException {
String historyFileInputPath="";
String deltaFileDirectoryPath="";
String pureDeltaFileOutPath="";
Configuration config= new Configuration();
Job job = new Job(config, "Pure Delta File Creation");
job.setJarByClass(CreatePureDeltaDriver.class);
Path historyDirPath= new Path(historyFileInputPath);
FileSystem fs = FileSystem.get(config);
FileStatus[] statusHistory = fs.listStatus(historyDirPath);
for (FileStatus fStatus : statusHistory) {
String historyFileName=fStatus.getPath().getName();
if(historyFileName.contains("part-r")){
MultipleInputs.addInputPath(job, fStatus.getPath(), TextInputFormat.class,CreatePureDeltaMapperOne.class);
}
}
Path deltaDirPath= new Path(deltaFileDirectoryPath);
FileStatus[] statusDelta = fs.listStatus(deltaDirPath);
for (FileStatus fStatus : statusDelta) {
String deltaFileName=fStatus.getPath().getName();
if(deltaFileName.startsWith("part-r")){
MultipleInputs.addInputPath(job, fStatus.getPath(), TextInputFormat.class, CreatePureDeltaMapperTwo.class);
}
}
job.setMapperClass(CreatePureDeltaMapperOne.class);
job.setMapperClass(CreatePureDeltaMapperTwo.class);
job.setReducerClass(CreatePureDeltaReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
Path hisInPath = new Path(historyFileInputPath);
Path outPath = new Path(pureDeltaFileOutPath);
//MultipleInputs.addInputPath(job, hisInPath, TextInputFormat.class, CreatePureDeltaMapperOne.class);
//MultipleInputs.addInputPath(job, delPath, TextInputFormat.class, CreatePureDeltaMapperTwo.class);
FileOutputFormat.setOutputPath(job, outPath);
System.out.println(job.waitForCompletion(true));
}
}
MY MAPPER CLASS
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Mapper.Context;
public class CreatePureDeltaMapperOne extends Mapper<LongWritable, Text, Text, Text> {
private Text outKey = new Text();
private Text outValue = new Text();
int counter=0;
private String delimiter="";
private int primaryKeyIndicator =0;
private Integer numMapNodes = null;
public void setup(Context context) throws IOException{
System.out.println("SETUP--- Mapper 1");
Configuration config = context.getConfiguration();
Properties properties = new Properties();
String propertyDirectory = config.get("propertyDirectory");
String propertyFileName =config.get("propertyFileName");
Path propertyDirPath= new Path(propertyDirectory);
FileSystem fs = FileSystem.get(config);
FileStatus[] status = fs.listStatus(propertyDirPath);
for (FileStatus fStatus : status) {
String propFileName=fStatus.getPath().getName().trim();
if(propFileName.equals(propertyFileName)){
properties.load(new InputStreamReader(fs.open(fStatus.getPath())));
this.setNumMapNodes(Integer.parseInt(properties.getProperty("num.of.nodes").trim()));
this.setDelimiter(properties.getProperty("file.delimiter.type").trim());
this.setPrimaryKeyIndicator(Integer.parseInt(properties.getProperty("file.primary.key.index.specifier").trim()));
}
}
}
public void map(LongWritable key, Text val, Context context) throws IOException, InterruptedException{
String valueString = val.toString().trim();
String[] tokens = valueString.split(this.getDelimiter());
String temp=tokens[this.getPrimaryKeyIndicator()].toString();
System.out.println(" MAPPER 1 invoked");
this.setOutKey(new Text(tokens[this.getPrimaryKeyIndicator()].toString().trim()));//Account number
this.setOutValue(new Text("h"+valueString.trim()));
context.write(outKey,outValue );
}
}

Answer 1

Do not use these two lines in your code :
job.setMapperClass(CreatePureDeltaMapperOne.class);
job.setMapperClass(CreatePureDeltaMapperTwo.class);
Because you are already passing name of corresponding class in the loop.
Hope it helps..