EnableMonitoringZ.scala

/*
 *  This works really well with the dockerhub image @
 *  https://hub.docker.com/r/kamon/grafana_graphite/
 *  To build on Bluemix IBM Containers, simply execute this command
 *  cf ic cpi kamon/grafana_graphite registry.ng.bluemix.net/goodenough/grafana_graphite:new
 *
 *  Be sure to expose the following public ports when creating the container
 *  80/tcp, 81/tcp, 2003/tcp, 2004/tcp, 7002/tcp, 8125/udp, 8126/tcp
 *  https://github.com/kamon-io/docker-grafana-graphite/blob/master/graphite/carbon.conf#L31-#L37
 *
 *  To obtain the dv-jdbc-3.1.22510.jar Rocket Driver dependency, obtain from Rocket Software.
 *  To obtain the log4j-api-2.7.jar dependency, fetch from maven.
 *  e.g. wget http://central.maven.org/maven2/org/apache/logging/log4j/log4j-api/2.7/log4j-api-2.7.jar -O log4j-api-2.7.jar
 */
import org.apache.spark.{SparkContext, SparkConf}
def enableMonitoring(sc: org.apache.spark.SparkContext) : org.apache.spark.SparkContext = {
    val conf = new SparkConf()
    sc.getConf.getAll.foreach {
        case ("spark.app.id", value) => {
        }
        case (key, value) => {
            conf.set(key, value)
        }
        case _ => 
    }
    conf.setAppName("sparkmon")
    conf.set("spark.metrics.conf.*.sink.graphite.class","org.apache.spark.metrics.sink.GraphiteSink")
    conf.set("spark.metrics.conf.*.sink.graphite.host","169.44.14.96")
    conf.set("spark.metrics.conf.*.sink.graphite.port","2003")
    conf.set("spark.metrics.conf.*.sink.graphite.period","10")
    conf.set("spark.metrics.conf.*.sink.graphite.unit","seconds")
    conf.set("spark.metrics.conf.*.sink.graphite.prefix","smf")
    conf.set("spark.metrics.conf.master.source.jvm.class","org.apache.spark.metrics.source.JvmSource")
    conf.set("spark.metrics.conf.worker.source.jvm.class","org.apache.spark.metrics.source.JvmSource")
    conf.set("spark.metrics.conf.driver.source.jvm.class","org.apache.spark.metrics.source.JvmSource")
    conf.set("spark.metrics.conf.executor.source.jvm.class","org.apache.spark.metrics.source.JvmSource")
    conf.set("spark.metrics.conf.application.source.jvm.class","org.apache.spark.metrics.source.JvmSource")
    conf.set("spark.metrics.conf.driver.source.system.class","org.apache.spark.metrics.source.SystemMetrics")
    conf.set("spark.metrics.conf.driver.source.proc.class","org.apache.spark.metrics.source.ProcMetrics")
    conf.set("spark.default.parallelism","20")
    conf.set("spark.eventLog.enabled","true")
    conf.set("spark.executor.instances","2")
    conf.set("spark.rdd.compress","true")
    conf.set("spark.executor.memory","2g")
    conf.set("spark.storage.memoryFraction",".6") 
    conf.set("spark.core.connection.ack.wait.timeout","600")
    conf.setJars(Array("file:/opt/sparkkernel/lib/kernel-assembly-0.1.5-SNAPSHOT.jar",
                       "file:/home/jovyan/bin/dv-jdbc-3.1.22510.jar",
                       "file:/home/jovyan/bin/log4j-api-2.7.jar"))
    sc.stop()
    return new SparkContext(conf)
}