Skip to content

Instantly share code, notes, and snippets.

@sdpatil
Last active August 29, 2015 13:57
Show Gist options
  • Select an option

  • Save sdpatil/9626929 to your computer and use it in GitHub Desktop.

Select an option

Save sdpatil/9626929 to your computer and use it in GitHub Desktop.
Run hadoop program maven build file
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.spnotes.hadoop</groupId>
<artifactId>HadoopWordCount</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<hadoop.version>2.0.0-cdh4.0.0</hadoop.version>
<scp.user>cloudera</scp.user>
<scp.password>cloudera</scp.password>
<scp.host>172.16.225.176</scp.host>
<scp.dirCopyTo>/home/cloudera/test</scp.dirCopyTo>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>2.0.0-mr1-cdh4.0.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.10</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.mrunit</groupId>
<artifactId>mrunit</artifactId>
<version>1.0.0</version>
<classifier>hadoop2</classifier>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<finalName>Sorting</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.1</version>
<configuration>
<source>1.6</source>
<target>1.6</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<archive>
<manifest>
<mainClass>com.spnotes.hadoop.WordCountDriver</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<configuration>
<tasks>
<scp todir="${scp.user}:${scp.password}@${scp.host}:/${scp.dirCopyTo}"
file="${project.build.directory}/${project.build.finalName}.jar"
trust="true" failonerror="false">
</scp>
<sshexec host="${scp.host}" username="${scp.user}"
password="${scp.password}"
command="hadoop jar ${scp.dirCopyTo}/${project.build.finalName}.jar books/dickens.txt wordcount/outputs2" />
</tasks>
</configuration>
<dependencies>
<dependency>
<groupId>ant</groupId>
<artifactId>ant-jsch</artifactId>
<version>1.6.5</version>
</dependency>
<dependency>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
<version>0.1.42</version>
</dependency>
</dependencies>
</plugin>
</plugins>
</build>
<repositories>
<repository>
<id>Apache Repository</id>
<url>https://repository.apache.org/content/repositories</url>
</repository>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos</url>
</repository>
</repositories>
</project>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment