Created
August 16, 2012 05:34
-
-
Save suisho/3367138 to your computer and use it in GitHub Desktop.
Output extension ".tsv" for Hadoop
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import java.io.DataOutputStream; | |
import java.io.IOException; | |
import org.apache.hadoop.conf.Configuration; | |
import org.apache.hadoop.fs.FileSystem; | |
import org.apache.hadoop.fs.Path; | |
import org.apache.hadoop.fs.FSDataOutputStream; | |
import org.apache.hadoop.io.compress.CompressionCodec; | |
import org.apache.hadoop.io.compress.GzipCodec; | |
import org.apache.hadoop.mapreduce.RecordWriter; | |
import org.apache.hadoop.mapreduce.TaskAttemptContext; | |
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; | |
import org.apache.hadoop.util.*; | |
/** | |
* Output file extension to tsv | |
*/ | |
public class TsvOutputFormat<K,V> extends TextOutputFormat<K, V> { | |
@Override | |
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext job) throws IOException, InterruptedException { | |
Configuration conf = job.getConfiguration(); | |
boolean isCompressed = getCompressOutput(job); | |
String keyValueSeparator= conf.get("mapred.textoutputformat.separator", | |
"\t"); | |
CompressionCodec codec = null; | |
String extension = ".tsv"; | |
if (isCompressed) { | |
Class<? extends CompressionCodec> codecClass = | |
getOutputCompressorClass(job, GzipCodec.class); | |
codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf); | |
extension = codec.getDefaultExtension(); | |
} | |
Path file = getDefaultWorkFile(job, extension); | |
FileSystem fs = file.getFileSystem(conf); | |
if (!isCompressed) { | |
FSDataOutputStream fileOut = fs.create(file, false); | |
return new LineRecordWriter<K, V>(fileOut, keyValueSeparator); | |
} else { | |
FSDataOutputStream fileOut = fs.create(file, false); | |
return new LineRecordWriter<K, V>(new DataOutputStream | |
(codec.createOutputStream(fileOut)), | |
keyValueSeparator); | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment