EnableMonitoring.scala

/*
 *  This works really well with the dockerhub image @
 *  https://hub.docker.com/r/kamon/grafana_graphite/
 *  To build on Bluemix IBM Containers, simply execute this command
 *  cf ic cpi kamon/grafana_graphite registry.ng.bluemix.net/goodenough/grafana_graphite:new
 *
 *  Be sure to expose the following public ports when creating the container
 *  80/tcp, 81/tcp, 2003/tcp, 2004/tcp, 7002/tcp, 8125/udp, 8126/tcp
 *  https://github.com/kamon-io/docker-grafana-graphite/blob/master/graphite/carbon.conf#L31-#L37
 */
import org.apache.spark.{SparkContext, SparkConf}
def enableMonitoring(sc: org.apache.spark.SparkContext) : org.apache.spark.SparkContext = {
    val conf = new SparkConf()
    sc.getConf.getAll.foreach {
        case (key, value) => {
            conf.set(key, value)
        }
        case _ => 
    }
    conf.setAppName("sparkmon")
    conf.set("spark.metrics.conf.*.sink.graphite.class","org.apache.spark.metrics.sink.GraphiteSink")
    conf.set("spark.metrics.conf.*.sink.graphite.host","Graphite_HOSTNAME")
    conf.set("spark.metrics.conf.*.sink.graphite.port","2003")
    conf.set("spark.metrics.conf.*.sink.graphite.period","10")
    conf.set("spark.metrics.conf.*.sink.graphite.unit","seconds")
    conf.set("spark.metrics.conf.*.sink.graphite.prefix","ibmjstart")
    conf.set("spark.metrics.conf.master.source.jvm.class","org.apache.spark.metrics.source.JvmSource")
    conf.set("spark.metrics.conf.worker.source.jvm.class","org.apache.spark.metrics.source.JvmSource")
    conf.set("spark.metrics.conf.driver.source.jvm.class","org.apache.spark.metrics.source.JvmSource")
    conf.set("spark.metrics.conf.executor.source.jvm.class","org.apache.spark.metrics.source.JvmSource")
    conf.set("spark.metrics.conf.application.source.jvm.class","org.apache.spark.metrics.source.JvmSource")
    conf.set("spark.metrics.conf.driver.source.system.class","org.apache.spark.metrics.source.SystemMetrics")
    conf.set("spark.metrics.conf.driver.source.proc.class","org.apache.spark.metrics.source.ProcMetrics")    
    sc.stop()
    return new SparkContext(conf)
}