Skip to content

Instantly share code, notes, and snippets.

@arjones
Forked from rweald/ParseJsonExampleJob.scala
Created February 28, 2013 18:38
Show Gist options
  • Save arjones/5059016 to your computer and use it in GitHub Desktop.
Save arjones/5059016 to your computer and use it in GitHub Desktop.
import com.twitter.scalding._
import scala.util.parsing.json._
class ParseJsonJob(args: Args) extends Job(args) {
TextLine(args("input"))
.map(('line) -> ('parseStatus, 'uri)) {
line: String => {
JSON.parseFull(line) match {
case Some(data: Map[String, String]) => ("success", data("uri"))
case None => ("failed", "")
}
}
}
.filter('parseStatus) { status: String => status != "failed" }
.project('uri)
.write(Tsv(args("output")))
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment