Skip to content

Instantly share code, notes, and snippets.

@quintona
Created March 20, 2014 12:20
Show Gist options
  • Save quintona/9662544 to your computer and use it in GitHub Desktop.
Save quintona/9662544 to your computer and use it in GitHub Desktop.
Simpler Tool for scalding that doesn't use reflection
object OpsMain {
def main(args : Array[String]) {
hadoop.util.ToolRunner.run(new hadoop.mapred.JobConf, new Tool[XXX]({args => new XXX(args)}), args)
}
}
import org.apache.hadoop
import com.twitter.scalding.{Job, Args, Mode}
class Tool[T <: Job](val createJob:(Args) => T) extends hadoop.conf.Configured with hadoop.util.Tool {
protected def nonHadoopArgsFrom(args : Array[String]) : Array[String] = {
(new hadoop.util.GenericOptionsParser(getConf, args)).getRemainingArgs
}
def parseModeArgs(args : Array[String]) : (Mode, Args) = {
val a = Args(nonHadoopArgsFrom(args))
(Mode(a, getConf), a)
}
def run(args : Array[String]) : Int = {
val (mode, jobArgs) = parseModeArgs(args)
val job = createJob(Mode.putMode(mode, jobArgs))
if(job.run)0 else 1
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment