iPython Toree Notebook Config Output

Python Notebooks

import os
import sys
import subprocess
masterCores = !cat /proc/cpuinfo | grep cores | tail -1 | tail -c 2
masterCores = int(masterCores[0])
print("Spark Context Version: " + sc.version);
print ("Spark Scala Version: " + os.environ['SPARK_SCALA_VERSION'])
print("Python Release version: " + sys.version)
print(subprocess.check_output(["java", "-version"], stderr=subprocess.STDOUT))
print("_____________________")
print("Spark Context Config:")
print sc._conf.toDebugString()
print("spark.default.parallelism=%d" % sc.defaultParallelism)
print("_____________________");
print("Spark Master Cores");
print("(Master Cores: %d)" % (masterCores))

=========================================================================================

Scala Notebooks
/** Reference && Inspiration 
 *
 *  http://stackoverflow.com/questions/33725500/load-data-from-bluemix-object-store-in-spark
 *  https://developer.ibm.com/clouddataservices/start-developing-with-spark-and-notebooks/
 *  https://github.com/ibm-et/spark-kernel/wiki/List-of-Current-Magics-for-the-Spark-Kernel
 *
 */
import sys.process._
import org.apache.spark.sql.SQLContext
val sqlctx = new SQLContext(sc)
val masterCores = ("cat /proc/cpuinfo" #| "grep cores" #| "tail -1" #| "tail -c 3").!!.trim
println("Spark Context Version: " + sc.version);
println("Java version: " + scala.util.Properties.javaVersion);
println("Scala Release version: " + scala.util.Properties.releaseVersion);
println("_____________________");
println("Spark Context Config:");
sc.getConf.getAll.foreach(println);
println("(spark.default.parallelism," + sc.defaultParallelism + ")")
println("_____________________");
println("Spark Master Cores");
println("(Master Cores: " + masterCores + ")")
println("_____________________");
println("Spark Context Executors:");
sc.getExecutorMemoryStatus.map(_._1).foreach(println)
println("_____________________");
println("Spark SQL Context Config:");
sqlctx.getAllConfs.foreach(println);