Skip to content

Instantly share code, notes, and snippets.

@c0mpiler
Created February 10, 2016 22:14
Show Gist options
  • Save c0mpiler/09a7d18ac60e85a027bb to your computer and use it in GitHub Desktop.
Save c0mpiler/09a7d18ac60e85a027bb to your computer and use it in GitHub Desktop.
ploading files through Java API to HDFS
package com.ins8s.hdfs.tools;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSClient;
public class HDFSUpload {
private static String hdfsUrl = "<your hdfs NameNode endpoint>";
private String sourceFilename;
private String destinationFilename;
public String getSourceFilename() {
return sourceFilename;
}
public void setSourceFilename(String sourceFilename) {
this.sourceFilename = sourceFilename;
}
public String getDestinationFilename() {
return destinationFilename;
}
public void setDestinationFilename(String destinationFilename) {
this.destinationFilename = destinationFilename;
}
public void uploadFile()
throws IOException, URISyntaxException {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", this.hdfsUrl);
DFSClient client = new DFSClient(new URI(this.hdfsUrl), conf);
OutputStream out = null;
InputStream in = null;
try {
if (client.exists(destinationFilename)) {
System.out.println("File already exists in hdfs: " + destinationFilename);
return;
}
out = new BufferedOutputStream(client.create(destinationFilename, false));
in = new BufferedInputStream(new FileInputStream(sourceFilename));
byte[] buffer = new byte[1024];
int len = 0;
while ((len = in.read(buffer)) > 0) {
out.write(buffer, 0, len);
}
} finally {
if (client != null) {
client.close();
}
if (in != null) {
in.close();
}
if (out != null) {
out.close();
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment