Skip to content

Instantly share code, notes, and snippets.

@MLnick
Created September 16, 2013 08:50
Show Gist options
  • Save MLnick/6578146 to your computer and use it in GitHub Desktop.
Save MLnick/6578146 to your computer and use it in GitHub Desktop.
trait Configured {
val config = ConfigFactory.load().getConfig("spark")
}
object ConfigUtils {
def asOption[T](t: => T): Option[T] = {
try {
Option(t)
} catch {
case e: ConfigException.Missing => None
case e: ConfigException => throw e
case e: Exception => throw e
}
}
}
class SparkConfig extends Configured {
val sparkHome = config.getString("home") // forcing types on instantiation
val localDir = config.getString("local_dir")
val bufferSize = config.getInt("buffer_size")
val kryoBufferSizeMb = config.getInt("kryo.buffer_size_mb")
...
val sparkMem = asOption { System.getEnv("SPARK_MEM") } getOrElse config.getString("worker.mem")
}
class SparkContext(..., config: SparkConfig) {
val home = config.sparkHome // type safe
...
}
{ "spark":
{
"home": null,
"local_dir": null,
"buffer_size": 65536,
"kryo": {
"buffer_size_mb": 10,
"registrator": null
},
...
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment