Created
November 16, 2017 02:51
-
-
Save ShanikaEdiriweera/0ad03bba82e2a4fbff121ef1aab78034 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import java.io.IOException; | |
import org.apache.hadoop.conf.Configuration; | |
import org.apache.hadoop.fs.Path; | |
import org.apache.hadoop.io.FloatWritable; | |
import org.apache.hadoop.io.Text; | |
import org.apache.hadoop.mapreduce.Job; | |
import org.apache.hadoop.mapreduce.Mapper; | |
import org.apache.hadoop.mapreduce.Reducer; | |
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; | |
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; | |
public class AverageGPA { | |
public static class TokenizerMapper | |
extends Mapper<Object, Text, Text, FloatWritable>{ | |
public void map(Object key, Text value, Context context | |
) throws IOException, InterruptedException { | |
String[] record = value.toString().split(","); | |
String ID = record[0]; | |
try { | |
Float gpa = Float.parseFloat(record[2]); | |
context.write(new Text(ID), new FloatWritable(gpa)); | |
} catch (Exception e) { | |
e.printStackTrace(); | |
} | |
} | |
} | |
public static class AverageReducer | |
extends Reducer<Text, FloatWritable, Text, Text> { | |
public void reduce(Text key, Iterable<FloatWritable> values, | |
Context context | |
) throws IOException, InterruptedException { | |
Float total = (float) 0; | |
int count = 0; | |
try { | |
for (FloatWritable var : values) { | |
total += var.get(); | |
System.out.println("reducer " + var.get()); | |
count++; | |
} | |
Float avg = (Float) total / count; | |
context.write(key, new Text(String.valueOf(avg))); | |
} catch (Exception e) { | |
e.printStackTrace(); | |
} | |
} | |
} | |
public static void main(String[] args) throws Exception { | |
Configuration conf = new Configuration(); | |
Job job = Job.getInstance(conf, "Average GPA"); | |
job.setJarByClass(AverageGPA.class); | |
job.setMapperClass(TokenizerMapper.class); | |
//job.setCombinerClass(AverageReducer.class); | |
job.setReducerClass(AverageReducer.class); | |
// Map output key, value types | |
job.setOutputKeyClass(Text.class); | |
job.setOutputValueClass(FloatWritable.class); | |
FileInputFormat.addInputPath(job, new Path(args[0])); | |
FileOutputFormat.setOutputPath(job, new Path(args[1])); | |
System.exit(job.waitForCompletion(true) ? 0 : 1); | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment