How to normalize columns in csv with hadoop - hadoop

Question

I want to read a csv file and normalize the data. If I understand well how hadoop works, the mapper gets the data line by line.
I found this formula to normalize : Xnew = (X - Xmin)/(Xmax - Xmin)
So I need to know the minimum value of the column and the maximum in order to normalize.
How can I do that when in a mapper I have access to only one line at a time ?

Answer 1

The problem with finding the max and min value of a column it this type of application is the scope of the max/min variables where they can be accessed/modified in a parallel program where each instance is isolated from the other in terms of data. So what it needs to be done here is finding a way to have a global scope for the max/min variables in order to access and synchronize their own instances at the end of each map/reduce step.
The closest thing to this supported by Hadoop (at the time this answer was written) is the feature of counters, but they are designed in a way to only increment their values so you have to be creative to achieve your desired output.
The trick here is to actually have if-statements modifying the maximum and minimum counters to the column value of each line (in case they are the max and/or min), by
resetting the counter to zero by adding the negative value of itself and then
increment the counter to the value of this specific line from the input csv file
It's a bit tedious, but it does the job inside the Map function.
Now, for accessing the max and min values of the counters from the Reduce function, we can simply get them in a setup method before the execution of all reducer instances and use them for computing the new normalized values of each key-value pair.
So, let's say we have a grades.csv file stored in the grades directory in the HDFS, which the grades of the students at an elementary school class are stored like that:
Jack,3
Dennis,5
Kate,10
Nancy,9
Peter,1
Zack,2
Alex,4
Yvonne,10
Violet,1
Claire,2
We can find the max and min values at the Map stage while turning each line of the input file into key-value pairs, and compute the normalized grade for each student (using the max and min values of course) at the Reduce stage as seen below:
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Cluster;
import java.io.*;
import java.io.IOException;
import java.util.*;
import java.nio.charset.StandardCharsets;
public class NormGrades
{
public static enum Global_Counters
{
MAX_GRADE,
MIN_GRADE
}
/* input: <byte_offset, line_of_tweet>
* output: <student, grade>
*/
public static class Map_Normalize extends Mapper<Object, Text, Text, IntWritable>
{
public void map(Object key, Text value, Context context) throws IOException, InterruptedException
{
String line = value.toString();
String[] columns = line.split(",");
int student_grade = Integer.parseInt(columns[1]);
int max_grade = Math.toIntExact(context.getCounter(Global_Counters.MAX_GRADE).getValue());
int min_grade = Math.toIntExact(context.getCounter(Global_Counters.MIN_GRADE).getValue());
// in order to find the maximum grade, we first set the max grade counter to 0
// by "increasing" it to the negative value of itself, and then increment by
// the new found maximum grade
if(student_grade > max_grade)
{
context.getCounter(Global_Counters.MAX_GRADE).increment(max_grade*(-1));
context.getCounter(Global_Counters.MAX_GRADE).increment(student_grade);
}
// in order to find the minimum grade, we first set the min grade counter to 0
// by "increasing" it to the negative value of itself, and then increment by
// the new found minimum grade
// the contents on this if statement will be accessed at least once in order to
// make sure that the min grade counter value is certainly higher than 0
if((student_grade < min_grade) || (min_grade == 0))
{
context.getCounter(Global_Counters.MIN_GRADE).increment(min_grade*(-1));
context.getCounter(Global_Counters.MIN_GRADE).increment(student_grade);
}
context.write(new Text(columns[0]), new IntWritable(student_grade));
}
}
/* input: <student, grade>
* output: <student, normalized_grade>
*/
public static class Reduce_Normalize extends Reducer<Text, IntWritable, Text, DoubleWritable>
{
public int max_grade, min_grade;
protected void setup(Context context) throws IOException, InterruptedException
{
Configuration conf = context.getConfiguration();
Cluster cluster = new Cluster(conf);
Job current_job = cluster.getJob(context.getJobID());
max_grade = Math.toIntExact(current_job.getCounters().findCounter(Global_Counters.MAX_GRADE).getValue());
min_grade = Math.toIntExact(current_job.getCounters().findCounter(Global_Counters.MIN_GRADE).getValue());
}
public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException
{
// each reducer instance is run for each student, so there is only one value/grade to access
int student_grade = values.iterator().next().get();
Double normalized_grade = (double) (student_grade - min_grade) / (max_grade - min_grade);
context.write(key, new DoubleWritable(normalized_grade));
}
}
public static void main(String[] args) throws Exception
{
Path input_dir = new Path("grades");
Path output_dir = new Path("normalized_grades");
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
if(fs.exists(output_dir))
fs.delete(output_dir, true);
Job normalize_job = Job.getInstance(conf, "Normalize Grades");
normalize_job.setJarByClass(NormGrades.class);
normalize_job.setMapperClass(Map_Normalize.class);
normalize_job.setReducerClass(Reduce_Normalize.class);
normalize_job.setMapOutputKeyClass(Text.class);
normalize_job.setMapOutputValueClass(IntWritable.class);
normalize_job.setOutputKeyClass(Text.class);
normalize_job.setOutputValueClass(DoubleWritable.class);
TextInputFormat.addInputPath(normalize_job, input_dir);
TextOutputFormat.setOutputPath(normalize_job, output_dir);
normalize_job.waitForCompletion(true);
}
}
The results are being stored as seen through the HDFS Browser in the following screenshot: