Last active
January 1, 2016 12:59
-
-
Save ktoso/8148507 to your computer and use it in GitHub Desktop.
Usage example of: https://github.com/ktoso/hadoop-scalding-nojartool
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pl.project13.hadoop.scalding.NoJarTool | |
import com.twitter.scalding | |
import org.apache.hadoop.util.ToolRunner | |
import org.apache.hadoop.conf.Configuration | |
val conf = new Configuration | |
val masterIp = "10.0.0.1" | |
// make suse these are set, otherwise Cascading will use "LocalJobRunner" | |
conf.setStrings("fs.default.name", s"hdfs://$masterIp:9000") | |
conf.setStrings("mapred.job.tracker", s"$masterIp:9001") | |
val tool = new NoJarTool( | |
wrappedTool = new scalding.Tool, | |
collectClassesFrom = Some(new File("target/scala-2.10/classes")), | |
libJars = List(new File("target/scala-2.10/all-deps.jar")) | |
) | |
ToolRunner.run(conf, tool, argsWithName) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment