I dont know what I'm doing wrong. When I hit control-z, I expected the "done" statement to be printed on the screen. But it does not print it.Please let me know why.
import java.io.*;
import java.lang.*;
import java.util.*;
class Cdt
{
public static void main(String args[])
{
String s=new String();
try{
BufferedReader br=new BufferedReader(new InputStreamReader(System.in));
while (s!=null)
{
s=br.readLine();
System.out.println(s);
}
System.out.println("Done");
}catch(IOException e){}
}
}
Related
I am getting java.lang.NoSuchMethodException please help me in this ...
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class Placeholder extends Configured implements Tool
{
private static Path[] filepath;
private static String filename;
private static String[] calendartablecolumn;
private static BufferedReader br;
private static String placeholder;
private static String output;
private static HashMap<String,String> map = new HashMap<String,String>();
public int run(String[] args) throws Exception {
String placeholdername = args[3];
String stratposition = args[4];
String length = args[5];
String category = args[6];
String calendarid = args[7];
Configuration conf = new Configuration();
conf.set("placeholdername", placeholdername);
conf.set("stratposition", stratposition);
conf.set("length", length);
conf.set("category", category);
conf.set("calendarid", calendarid);
Job job = Job.getInstance(conf, "Placeholder");
DistributedCache.addCacheFile(new Path(args[2]).toUri(), getConf());
job.setJarByClass(Placeholder.class);
job.setMapperClass(PlaceholderMapper.class);
job.setNumReduceTasks(0);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
return job.waitForCompletion(true) ? 0 : 1;
}
public static void main(String[] args) throws Exception {
int res = ToolRunner.run(new Configuration(), new Placeholder(), args);
System.exit(res);
}
public class PlaceholderMapper extends Mapper<LongWritable,Text,Text,Text>
{
protected void setup(Context context) throws IOException,InterruptedException
{
try {
filepath = DistributedCache.getLocalCacheFiles(getConf());
filename = filepath[0].getName().toString();
br = new BufferedReader(new FileReader(filepath[0].toString()));
String line = null;
while ((line = br.readLine()) != null)
{
calendartablecolumn = line.split("\\,");
}
} catch (Exception e) {
}
}
public void map(LongWritable keys, Text value, Context context) throws IOException, InterruptedException
{
String str_placeholdername = context.getConfiguration().get("placeholdername");
String str_stratposition = context.getConfiguration().get("stratposition");
int int_stratposition = Integer.parseInt(str_stratposition);
String str_length = context.getConfiguration().get("length");
int int_length = Integer.parseInt(str_length);
String str_category = context.getConfiguration().get("category");
String str_calendarid = context.getConfiguration().get("calendarid");
int int_calendarid = Integer.parseInt(str_calendarid);
if (str_category != null)
{
if (str_category.equalsIgnoreCase("filename"))
{
placeholder = filename.substring((int_stratposition-1),(int_length-1));
}else if (str_category.equalsIgnoreCase("startdate"))
{
map.put(calendartablecolumn[0].trim(),calendartablecolumn[3].trim());
placeholder = map.get(int_calendarid);
}else if (str_category.equalsIgnoreCase("enddate"))
{
map.put(calendartablecolumn[0].trim(),calendartablecolumn[4].trim());
placeholder = map.get(int_calendarid);
}else if (str_category.equalsIgnoreCase("currentdate"))
{
DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
Date date = new Date();
String currentdate = (dateFormat.format(date)).toString();
placeholder = currentdate.substring((int_stratposition-1),(int_length-1));
}else if (str_category.equalsIgnoreCase("isdailydata"))
{
}
output = value+","+placeholder;
}else
{
String line = value.toString();
output = line.substring((int_stratposition-1),(int_length-1));
}
context.write(null,new Text (output));
}
}
}
This is the error that i am getting-
Error: java.lang.RuntimeException: java.lang.NoSuchMethodException:
com.nielsen.GRFE.processor.mapreduce.Placeholder$PlaceholderMapper.()
at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:131)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:745)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:163)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158) Caused by: java.lang.NoSuchMethodException:
com.nielsen.GRFE.processor.mapreduce.Placeholder$PlaceholderMapper.()
at java.lang.Class.getConstructor0(Class.java:2849)
at java.lang.Class.getDeclaredConstructor(Class.java:2053)
at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:125)
I think the problem is that your PlaceholderMapper class is not static, try to make it public static.
I am new to hadoop mapreduce.I am trying to implement search in map reduce so my input file is like this
key1 value1,value3
key2 value2,value6
I want to find values list for key which user will pass as command line argument.for this my main (driver) class is like this
public static void main(String[] args) {
JobClient client = new JobClient();
JobConf conf = new JobConf(NameSearchJava.class);
// write now I am trying with writing search key in code (Joy),later I'll
//try to pass argument while running job from hadoop.
conf.set("searcKey", "Joy");
conf.setJobName("Search");
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(Text.class);
FileInputFormat.setInputPaths(conf, new Path(args[0]));
FileOutputFormat.setOutputPath(conf, new Path(args[1]));
conf.setMapperClass(SearchMapper.class);
conf.setReducerClass(SearchReducer.class);
client.setConf(conf);
try {
JobClient.runJob(conf);
} catch (Exception e) {
e.printStackTrace();
}
}
}
and my configure function is:
String item ;
public void configure(JobConf job) {
{
item = job.get("test");
System.out.println(item);
System.err.println("search" + item);
}
where should I write configure function in Mapper or Reducer.How can I use this item parameter to do comparison in reducer .Is this the correct way to take parameters in hadoop ?
Add on to Hadooper's Answer.
This is the full Code.
You can refer Hadooper's answer for explanation.
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
/**
* #author Unmesha sreeveni
* #Date 23 sep 2014
*/
public class StringSearchDriver extends Configured implements Tool {
public static class Map extends
Mapper<LongWritable, Text, Text, IntWritable> {
private final static IntWritable one = new IntWritable(1);
private Text word = new Text();
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
String line = value.toString();
String searchString = conf.get("word");
StringTokenizer tokenizer = new StringTokenizer(line);
while (tokenizer.hasMoreTokens()) {
String token = tokenizer.nextToken();
if(token.equals(searchString)){
word.set(token);
context.write(word, one);
}
}
}
}
public static class Reduce extends
Reducer<Text, IntWritable, Text, IntWritable> {
public void reduce(Text key, Iterable<IntWritable> values,
Context context) throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
context.write(key, new IntWritable(sum));
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
int res = ToolRunner.run(conf, new StringSearchDriver(), args);
System.exit(res);
}
#Override
public int run(String[] args) throws Exception {
// TODO Auto-generated method stub
if (args.length != 3) {
System.out
.printf("Usage: Search String <input dir> <output dir> <search word> \n");
System.exit(-1);
}
String source = args[0];
String dest = args[1];
String searchword = args[2];
Configuration conf = new Configuration();
conf.set("word", searchword);
Job job = new Job(conf, "Search String");
job.setJarByClass(StringSearchDriver.class);
FileSystem fs = FileSystem.get(conf);
Path in =new Path(source);
Path out =new Path(dest);
if (fs.exists(out)) {
fs.delete(out, true);
}
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
job.setMapperClass(Map.class);
job.setReducerClass(Reduce.class);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
FileInputFormat.addInputPath(job, in);
FileOutputFormat.setOutputPath(job, out);
boolean sucess = job.waitForCompletion(true);
return (sucess ? 0 : 1);
}
}
Read the command line argument in the Driver class as follows -
conf.set("searchKey", args[2]);
where args[2] will be the search-key passed as third argument.
The configure method should be coded in the Mapper as follows -
String searchWord;
public void configure(JobConf jc)
{
searchWord = jc.get("searchKey");
}
This will bring your key to be searched in the Mapper function.
You can perform the comparison in the Mapper itself using the logic as follows -
public void map(LongWritable key, Text value,
OutputCollector<Text, IntWritable> out, Reporter reporter)
throws IOException
{
String[] input = value.toString().split(" ");
for(String word:input)
{
if (word.equalsIgnoreCase(searchWord))
out.collect(new Text(word), new IntWritable(1));
}
}
Let me know if this helps!
I am new to Socket Programming and very well know that i am making a silly mistake. Unfortunately not able to figure out as to what is going on. I have created a basic multi Client-Server system using Java socket on same machine(shouldn't make any difference), however to my surprise either i am able to only
Send Message from Server to Client and then read it on Client.
OR
Send Message from Client to Server and then read it on Server.
What i am not able to achieve is
Send Message from Server to Client and then read it on Client.
AND
Send Message from Client to Server and then read it on Server.
I am attaching my code.
import java.io.IOException;
import java.net.ServerSocket;
public class ServerSocketMain {
public static void main(String[] args) throws IOException {
#SuppressWarnings("resource")
ServerSocket serverSocket = new ServerSocket(1234);
while(true){
System.out.println("Server Waiting for Client to Connect ....");
Thread thread = new Thread(new ServerReaderWriter(serverSocket.accept()));
System.out.println("Client connected....");
thread.start();
}
}
}
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.Socket;
public class ServerReaderWriter implements Runnable {
private Socket socket;
public ServerReaderWriter(Socket socket) {
this.socket = socket;
}
#Override
public void run() {
try {
readFromClient();
writeToClient();
} catch (IOException e) {
e.printStackTrace();
}
}
private void writeToClient() throws IOException {
DataOutputStream writeToClient = new DataOutputStream(this.socket.getOutputStream());
writeToClient.writeBytes("Communication from Client Thread "
+ Thread.currentThread().getName() + "\n");
}
private void readFromClient() throws IOException {
BufferedReader inFromClient = new BufferedReader(new InputStreamReader(
this.socket.getInputStream()));
System.out.println("Client Communication : " + inFromClient.readLine());
}
}
import java.io.IOException;
import java.net.Socket;
import java.net.UnknownHostException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
public class ClientSocketMain {
public static void main(String[] args) throws UnknownHostException, IOException {
ExecutorService executorService = Executors.newFixedThreadPool(10);
for (int i = 0; i < 10; i++) {
executorService.submit(new ClientReaderWriter(new Socket("localhost", 1234)));
}
executorService.shutdown();
}
}
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.Socket;
public class ClientReaderWriter implements Runnable {
private Socket socket;
public ClientReaderWriter(Socket socket) {
this.socket = socket;
}
#Override
public void run() {
try {
readFromServer(socket.getInputStream());
writeToServer(socket.getOutputStream());
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
socket.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
private void writeToServer(OutputStream outputStream) throws IOException {
DataOutputStream writeToServer = new DataOutputStream(outputStream);
writeToServer.writeBytes("Communication from Client Thread "
+ Thread.currentThread().getName() + "\n");
}
private void readFromServer(InputStream inputStream) throws IOException {
BufferedReader inFromServer = new BufferedReader(new InputStreamReader(
inputStream));
System.out.println(inFromServer.readLine());
}
}
I am just wondering if my client socket closes even before read and write operations are done..not sure just guessing out loud. please help!!
As it turned out...i was making a silly mistake....Ordering is very important in the code snippet that i had shared above and i was ignoring the fact.
readFromClient();
writeToClient();
readFromServer(socket.getInputStream());
writeToServer(socket.getOutputStream());
In the above scenario both server and client are trying to read...instead one should be writing and other reading. So all i needed to do was change the order
writeToClient();
readFromClient();
readFromServer(socket.getInputStream());
writeToServer(socket.getOutputStream());
Please help me in this code. I am trying to reiad images from HDFS. I am using WholeFileInputFormat. with WholeFileRecordreader. No compile time errors.But the code is giving runtime errors.
The output is saying: cannot create the instance of the given class WholeFileInputFormat.
I have written this code according to the comments on How to read multiple image files as input from hdfs in map-reduce?
Please help me in this code.It contains 3 classes.How to debug it? Or any other way?
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import javax.imageio.ImageIO;
import net.semanticmetadata.lire.imageanalysis.AutoColorCorrelogram;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.*;
import org.apache.hadoop.mapred.lib.NullOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class map2 extends Configured implements Tool {
public static class MapClass extends MapReduceBase
implements Mapper<NullWritable, BytesWritable, Text, Text> {
private Text input_image = new Text();
private Text input_vector = new Text();
#Override
public void map(NullWritable key,BytesWritable value,
OutputCollector<Text, Text> output,
Reporter reporter) throws IOException {
System.out.println("CorrelogramIndex Method:");
String featureString;
int MAXIMUM_DISTANCE = 16;
AutoColorCorrelogram.Mode mode = AutoColorCorrelogram.Mode.FullNeighbourhood;
byte[] identifier=value.getBytes();
BufferedImage bimg = ImageIO.read(new ByteArrayInputStream(identifier));
AutoColorCorrelogram vd = new AutoColorCorrelogram(MAXIMUM_DISTANCE, mode);
vd.extract(bimg);
featureString = vd.getStringRepresentation();
double[] bytearray = vd.getDoubleHistogram();
System.out.println("image: " + identifier + " " + featureString);
System.out.println(" ------------- ");
input_image.set(identifier);
input_vector.set(featureString);
output.collect(input_image, input_vector);
}
}
public static class Reduce extends MapReduceBase
implements Reducer<Text, Text, Text, Text> {
#Override
public void reduce(Text key, Iterator<Text> values,
OutputCollector<Text, Text> output,
Reporter reporter) throws IOException {
String out_vector = "";
while (values.hasNext()) {
out_vector += (values.next().toString());
}
output.collect(key, new Text(out_vector));
}
}
static int printUsage() {
System.out.println("map2 [-m <maps>] [-r <reduces>] <input> <output>");
ToolRunner.printGenericCommandUsage(System.out);
return -1;
}
#Override
public int run(String[] args) throws Exception {
JobConf conf = new JobConf(getConf(), map2.class);
conf.setJobName("image_mapreduce");
conf.setInputFormat(WholeFileInputFormat.class);
conf.setOutputFormat(NullOutputFormat.class);
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(Text.class);
conf.setMapperClass(MapClass.class);
conf.setReducerClass(Reduce.class);
List<String> other_args = new ArrayList<>();
for (int i = 0; i < args.length; ++i) {
try {
switch (args[i]) {
case "-m":
conf.setNumMapTasks(Integer.parseInt(args[++i]));
break;
case "-r":
conf.setNumReduceTasks(Integer.parseInt(args[++i]));
break;
default:
other_args.add(args[i]);
break;
}
} catch (NumberFormatException except) {
System.out.println("ERROR: Integer expected instead of " + args[i]);
return printUsage();
} catch (ArrayIndexOutOfBoundsException except) {
System.out.println("ERROR: Required parameter missing from "
+ args[i - 1]);
return printUsage();
}
}
// Make sure there are exactly 2 parameters left.
if (other_args.size() != 2) {
System.out.println("ERROR: Wrong number of parameters: "
+ other_args.size() + " instead of 2.");
return printUsage();
}
FileInputFormat.setInputPaths(conf, other_args.get(0));
FileOutputFormat.setOutputPath(conf, new Path(other_args.get(1)));
JobClient.runJob(conf);
return 0;
}
public static void main(String[] args) throws Exception {
int res = ToolRunner.run(new Configuration(), new map2(), args);
System.exit(res);
}
}
-----------------------------------------------------------------------------------
//WholeFileInputFormat
import java.io.IOException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.*;
public class WholeFileInputFormat<NullWritable, BytesWritable>
extends FileInputFormat<NullWritable, BytesWritable> {
// #Override
protected boolean isSplitable(JobContext context, Path file) {
return false;
}
//#Override
public WholeFileRecordReader createRecordReader(
InputSplit split, TaskAttemptContext context) throws IOException,
InterruptedException {
WholeFileRecordReader reader = new WholeFileRecordReader();
reader.initialize(split, context);
return reader;
}
#Override
public RecordReader<NullWritable, BytesWritable> getRecordReader(InputSplit split,
JobConf job, Reporter reporter)
throws IOException;
}
-------------------------------------------------------------------------------
//WholeInputFileRecorder
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.TaskAttemptContext;
class WholeFileRecordReader implements RecordReader<NullWritable, BytesWritable> { //recordreader
private FileSplit fileSplit;
private Configuration conf;
private BytesWritable value = new BytesWritable();
private boolean processed = false;
public void initialize(InputSplit split, TaskAttemptContext context)
throws IOException, InterruptedException {
this.fileSplit = (FileSplit) split;
this.conf = context.getJobConf();
}
#Override
public boolean next(NullWritable k, BytesWritable v) throws IOException {
if (!processed) {
byte[] contents = new byte[(int) fileSplit.getLength()];
Path file = fileSplit.getPath();
org.apache.hadoop.fs.FileSystem fs = file.getFileSystem(conf);
FSDataInputStream in = null;
try {
in = fs.open(file);
IOUtils.readFully(in, contents, 0, contents.length);
value.set(contents, 0, contents.length);
} finally {
IOUtils.closeStream(in);
}
processed = true;
return true;
}
return false;
}
#Override
public NullWritable createKey() {
return NullWritable.get();
}
#Override
public BytesWritable createValue() {
return value;
}
#Override
public long getPos() throws IOException {
throw new UnsupportedOperationException("Not supported yet.");
}
#Override
public void close() throws IOException {
throw new UnsupportedOperationException("Not supported yet.");
}
#Override
public float getProgress() throws IOException {
throw new UnsupportedOperationException("Not supported yet.");
}
}
WholeFileInputFormat is defined as abstract, how do you want to create an instance of it?
Either make it not abstract or subclass it with a concrete implementation.
I am currently trying to figure out when you run a MapReduce job what happens by making some system.out.println() at certain places on the code but know of those print statement gets to print on my terminal when the job runs. Can someone help me out figure out what exactly am i doing wrong here.
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.StatusReporter;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class WordCountJob {
public static int iterations;
public static class TokenizerMapper
extends Mapper<Object, Text, Text, IntWritable>{
private final static IntWritable one = new IntWritable(1);
private Text word = new Text();
#Override
public void map(Object key, Text value, Context context
) throws IOException, InterruptedException {
System.out.println("blalblbfbbfbbbgghghghghghgh");
StringTokenizer itr = new StringTokenizer(value.toString());
while (itr.hasMoreTokens()) {
word.set(itr.nextToken());
String myWord = itr.nextToken();
int n = 0;
while(n< 5){
myWord = myWord+ "Test my appending words";
n++;
}
System.out.println("Print my word: "+myWord);
word.set(myWord);
context.write(word, one);
}
}
}
public static class IntSumReducer
extends Reducer<Text,IntWritable,Text,IntWritable> {
private IntWritable result = new IntWritable();
public void reduce(Text key, Iterable<IntWritable> values,
Context context
) throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
result.set(sum);
context.write(key, result);
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
TaskAttemptID taskid = new TaskAttemptID();
TokenizerMapper my = new TokenizerMapper();
if (args.length != 3) {
System.err.println("Usage: WordCountJob <in> <out> <iterations>");
System.exit(2);
}
iterations = new Integer(args[2]);
Path inPath = new Path(args[0]);
Path outPath = null;
for (int i = 0; i<iterations; ++i){
System.out.println("Iteration number: "+i);
outPath = new Path(args[1]+i);
Job job = new Job(conf, "WordCountJob");
job.setJarByClass(WordCountJob.class);
job.setMapperClass(TokenizerMapper.class);
job.setCombinerClass(IntSumReducer.class);
job.setReducerClass(IntSumReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job, inPath);
FileOutputFormat.setOutputPath(job, outPath);
job.waitForCompletion(true);
inPath = outPath;
}
}
}
This depends on how you are submitting your job, I think you're submitting it using bin/hadoop jar yourJar.jar right?
Your System.out.println() is only available in your main method, that is because the mapper/reducer is executed inside of hadoop in a different JVM, all outputs are redirected to special log files (out/log-files).
And I would recommend to use your own Apache-commons log using:
Log log = LogFactory.getLog(YOUR_MAPPER_CLASS.class)
And therefore do some info logging:
log.info("Your message");
If you're in "local"-mode then you can see this log in your shell, otherwise this log will be stored somewhere on the machine where the task gets executed. Please use the jobtracker's web UI to look at these log files, it is quite convenient. By default the job tracker runs on port 50030.
Alternatively, you can make use of MultipleOutputs class and re-direct all your log data into one output file(log).
MultipleOutputs<Text, Text> mos = new MultipleOutputs<Text, Text>(context);
Text tKey = new Text("key");
Text tVal = new Text("log message");
mos.write(tKey, tVal, <lOG_FILE>);