spark-notebook

#!/usr/bin/env bash

###  ------------------------------- ###
###  Helper methods for BASH scripts ###
###  ------------------------------- ###

die() {
  echo "$@" 1>&2
  exit 1
}

realpath () {
(
  TARGET_FILE="$1"
  CHECK_CYGWIN="$2"

  cd "$(dirname "$TARGET_FILE")"
  TARGET_FILE=$(basename "$TARGET_FILE")

  COUNT=0
  while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ]
  do
      TARGET_FILE=$(readlink "$TARGET_FILE")
      cd "$(dirname "$TARGET_FILE")"
      TARGET_FILE=$(basename "$TARGET_FILE")
      COUNT=$(($COUNT + 1))
  done

  if [ "$TARGET_FILE" == "." -o "$TARGET_FILE" == ".." ]; then
    cd "$TARGET_FILE"
    TARGET_FILEPATH=
  else
    TARGET_FILEPATH=/$TARGET_FILE
  fi

  # make sure we grab the actual windows path, instead of cygwin's path.
  if [[ "x$CHECK_CYGWIN" == "x" ]]; then
    echo "$(pwd -P)/$TARGET_FILE"
  else
    echo $(cygwinpath "$(pwd -P)/$TARGET_FILE")
  fi
)
}

# TODO - Do we need to detect msys?

# Uses uname to detect if we're in the odd cygwin environment.
is_cygwin() {
  local os=$(uname -s)
  case "$os" in
    CYGWIN*) return 0 ;;
    *)  return 1 ;;
  esac
}

# This can fix cygwin style /cygdrive paths so we get the
# windows style paths.
cygwinpath() {
  local file="$1"
  if is_cygwin; then
    echo $(cygpath -w $file)
  else
    echo $file
  fi
}

# Make something URI friendly
make_url() {
  url="$1"
  local nospaces=${url// /%20}
  if is_cygwin; then
    echo "/${nospaces//\\//}"
  else
    echo "$nospaces"
  fi
}

# This crazy function reads in a vanilla "linux" classpath string (only : are separators, and all /),
# and returns a classpath with windows style paths, and ; separators.
fixCygwinClasspath() {
  OLDIFS=$IFS
  IFS=":"
  read -a classpath_members <<< "$1"
  declare -a fixed_members
  IFS=$OLDIFS
  for i in "${!classpath_members[@]}"
  do
    fixed_members[i]=$(realpath "${classpath_members[i]}" "fix")
  done
  IFS=";"
  echo "${fixed_members[*]}"
  IFS=$OLDIFS
}

# Fix the classpath we use for cygwin.
fix_classpath() {
  cp="$1"
  if is_cygwin; then
    echo "$(fixCygwinClasspath "$cp")"
  else 
    echo "$cp"
  fi
}
# Detect if we should use JAVA_HOME or just try PATH.
get_java_cmd() {
  if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]];  then
    echo "$JAVA_HOME/bin/java"
  else
    echo "java"
  fi
}

echoerr () {
  echo 1>&2 "$@"
}
vlog () {
  [[ $verbose || $debug ]] && echoerr "$@"
}
dlog () {
  [[ $debug ]] && echoerr "$@"
}
execRunner () {
  # print the arguments one to a line, quoting any containing spaces
  [[ $verbose || $debug ]] && echo "# Executing command line:" && {
    for arg; do
      if printf "%s\n" "$arg" | grep -q ' '; then
        printf "\"%s\"\n" "$arg"
      else
        printf "%s\n" "$arg"
      fi
    done
    echo ""
  }

  # we use "exec" here for our pids to be accurate.
  exec "$@"
}
addJava () {
  dlog "[addJava] arg = '$1'"
  java_args+=( "$1" )
}
addApp () {
  dlog "[addApp] arg = '$1'"
  app_commands+=( "$1" )
}
addResidual () {
  dlog "[residual] arg = '$1'"
  residual_args+=( "$1" )
}
addDebugger () {
  addJava "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$1"
}

require_arg () {
  local type="$1"
  local opt="$2"
  local arg="$3"
  if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then
    die "$opt requires <$type> argument"
  fi
}
is_function_defined() {
  declare -f "$1" > /dev/null
}

# Attempt to detect if the script is running via a GUI or not
# TODO - Determine where/how we use this generically
detect_terminal_for_ui() {
  [[ ! -t 0 ]] && [[ "${#residual_args}" == "0" ]] && {
    echo "true"
  }
  # SPECIAL TEST FOR MAC
  [[ "$(uname)" == "Darwin" ]] && [[ "$HOME" == "$PWD" ]] && [[ "${#residual_args}" == "0" ]] && {
    echo "true"
  }
}

# Processes incoming arguments and places them in appropriate global variables.  called by the run method.
process_args () {
  local no_more_snp_opts=0
  while [[ $# -gt 0 ]]; do
    case "$1" in
             --) shift && no_more_snp_opts=1 && break ;;
       -h|-help) usage; exit 1 ;;
    -v|-verbose) verbose=1 && shift ;;
      -d|-debug) debug=1 && shift ;;

    -no-version-check) no_version_check=1 && shift ;;

           -mem) echo "!! WARNING !! -mem option is ignored. Please use -J-Xmx and -J-Xms" && shift 2 ;;
     -jvm-debug) require_arg port "$1" "$2" && addDebugger $2 && shift 2 ;;

          -main) custom_mainclass="$2" && shift 2 ;;

     -java-home) require_arg path "$1" "$2" && java_cmd="$2/bin/java" && shift 2 ;;

 -D*|-agentlib*) addJava "$1" && shift ;;
            -J*) addJava "${1:2}" && shift ;;
              *) addResidual "$1" && shift ;;
    esac
  done

  if [[ no_more_snp_opts ]]; then
    while [[ $# -gt 0 ]]; do
      addResidual "$1" && shift
    done
  fi

  is_function_defined process_my_args && {
    myargs=("${residual_args[@]}")
    residual_args=()
    process_my_args "${myargs[@]}"
  }
}

# Actually runs the script.
run() {
  # TODO - check for sane environment

  # process the combined args, then reset "$@" to the residuals
  process_args "$@"
  set -- "${residual_args[@]}"
  argumentCount=$#

  #check for jline terminal fixes on cygwin
  if is_cygwin; then
    stty -icanon min 1 -echo > /dev/null 2>&1
    addJava "-Djline.terminal=jline.UnixTerminal"
    addJava "-Dsbt.cygwin=true"
  fi
  
  # check java version
  if [[ ! $no_version_check ]]; then
    java_version_check
  fi
  
  if [ -n "$custom_mainclass" ]; then
    mainclass=("$custom_mainclass")
  else
    mainclass=("${app_mainclass[@]}")
  fi

  # Now we check to see if there are any java opts on the environment. These get listed first, with the script able to override them.
  if [[ "$JAVA_OPTS" != "" ]]; then
    java_opts="${JAVA_OPTS}"
  fi

  # run sbt
  execRunner "$java_cmd" \
    ${java_opts[@]} \
    "${java_args[@]}" \
    -cp "$(fix_classpath "$app_classpath")" \
    "${mainclass[@]}" \
    "${app_commands[@]}" \
    "${residual_args[@]}"
    
  local exit_code=$?
  if is_cygwin; then
    stty icanon echo > /dev/null 2>&1
  fi
  exit $exit_code
}

# Loads a configuration file full of default command line options for this script.
loadConfigFile() {
  cat "$1" | sed '/^\#/d'
}

# Now check to see if it's a good enough version
# TODO - Check to see if we have a configured default java version, otherwise use 1.6
java_version_check() { 
  readonly java_version=$("$java_cmd" -version 2>&1 | awk -F '"' '/version/ {print $2}')
  if [[ "$java_version" == "" ]]; then
    echo
    echo No java installations was detected.
    echo Please go to http://www.java.com/getjava/ and download
    echo
    exit 1
  elif [[ ! "$java_version" > "1.6" ]]; then
    echo
    echo The java installation you have is not up to date
    echo $app_name requires at least version 1.6+, you have
    echo version $java_version
    echo
    echo Please go to http://www.java.com/getjava/ and download
    echo a valid Java Runtime and install before running $app_name.
    echo
    exit 1
  fi
}

###  ------------------------------- ###
###  Start of customized settings    ###
###  ------------------------------- ###
usage() {
 cat <<EOM
Usage: $script_name [options]

  -h | -help         print this message
  -v | -verbose      this runner is chattier
  -d | -debug        set sbt log level to debug
  -no-version-check  Don't run the java version check.
  -main <classname>  Define a custom main class
  -jvm-debug <port>  Turn on JVM debugging, open at the given port.

  # java version (default: java from PATH, currently $(java -version 2>&1 | grep version))
  -java-home <path>         alternate JAVA_HOME

  # jvm options and output control
  JAVA_OPTS          environment variable, if unset uses "$java_opts"
  -Dkey=val          pass -Dkey=val directly to the java runtime
  -J-X               pass option -X directly to the java runtime
                     (-J is stripped)

  # special option
  --                 To stop parsing built-in commands from the rest of the command-line.
                     e.g.) enabling debug and sending -d as app argument
                     \$ ./start-script -d -- -d

In the case of duplicated or conflicting options, basically the order above
shows precedence: JAVA_OPTS lowest, command line options highest except "--".
EOM
}

###  ------------------------------- ###
###  Main script                     ###
###  ------------------------------- ###

declare -a residual_args
declare -a java_args
declare -a app_commands
declare -r real_script_path="$(realpath "$0")"
declare -r app_home="$(realpath "$(dirname "$real_script_path")")"
# TODO - Check whether this is ok in cygwin...
declare -r lib_dir="$(realpath "${app_home}/../lib")"
declare -a app_mainclass=("play.core.server.NettyServer")

declare -r script_conf_file="${app_home}/../conf/application.ini"
declare -r app_classpath="${CLASSPATH_OVERRIDES}:${YARN_CONF_DIR}:${HADOOP_CONF_DIR}:${EXTRA_CLASSPATH}:$lib_dir/nooostab.spark-notebook-0.6.2-scala-2.10.4-spark-1.5.2-hadoop-2.7.1-with-hive-with-parquet.jar:$lib_dir/tachyon.tachyon-0.6.2-scala-2.10.4-spark-1.5.2-hadoop-2.7.1-with-hive-with-parquet.jar:$lib_dir/subprocess.subprocess-0.6.2-scala-2.10.4-spark-1.5.2-hadoop-2.7.1-with-hive-with-parquet.jar:$lib_dir/observable.observable-0.6.2-scala-2.10.4-spark-1.5.2-hadoop-2.7.1-with-hive-with-parquet.jar:$lib_dir/common.common-0.6.2-scala-2.10.4-spark-1.5.2-hadoop-2.7.1-with-hive-with-parquet.jar:$lib_dir/spark.spark-0.6.2-scala-2.10.4-spark-1.5.2-hadoop-2.7.1-with-hive-with-parquet.jar:$lib_dir/kernel.kernel-0.6.2-scala-2.10.4-spark-1.5.2-hadoop-2.7.1-with-hive-with-parquet.jar:$lib_dir/wisp_2.10-0.0.5.jar:$lib_dir/org.scala-lang.scala-compiler-2.10.4.jar:$lib_dir/org.scala-lang.scala-library-2.10.4.jar:$lib_dir/org.scala-lang.scala-reflect-2.10.4.jar:$lib_dir/com.google.guava.guava-14.0.1.jar:$lib_dir/org.tachyonproject.tachyon-common-0.7.1.jar:$lib_dir/commons-io.commons-io-2.4.jar:$lib_dir/log4j.log4j-1.2.16.jar:$lib_dir/org.tachyonproject.tachyon-client-0.7.1.jar:$lib_dir/org.tachyonproject.tachyon-underfs-hdfs-0.7.1.jar:$lib_dir/org.tachyonproject.tachyon-underfs-local-0.7.1.jar:$lib_dir/org.tachyonproject.tachyon-servers-0.7.1.jar:$lib_dir/org.tachyonproject.tachyon-client-unshaded-0.7.1.jar:$lib_dir/org.eclipse.jetty.jetty-jsp-7.6.15.v20140411.jar:$lib_dir/org.eclipse.jetty.orbit.javax.servlet.jsp-2.1.0.v201105211820.jar:$lib_dir/org.eclipse.jetty.orbit.org.apache.jasper.glassfish-2.1.0.v201110031002.jar:$lib_dir/org.eclipse.jetty.orbit.javax.servlet.jsp.jstl-1.2.0.v201105211821.jar:$lib_dir/org.eclipse.jetty.orbit.org.apache.taglibs.standard.glassfish-1.2.0.v201112081803.jar:$lib_dir/org.eclipse.jetty.orbit.javax.el-2.1.0.v201105211819.jar:$lib_dir/org.eclipse.jetty.orbit.com.sun.el-1.0.0.v201105211818.jar:$lib_dir/org.eclipse.jetty.orbit.org.eclipse.jdt.core-3.7.1.jar:$lib_dir/org.eclipse.jetty.jetty-webapp-7.6.15.v20140411.jar:$lib_dir/org.eclipse.jetty.jetty-xml-7.6.15.v20140411.jar:$lib_dir/org.eclipse.jetty.jetty-util-7.6.15.v20140411.jar:$lib_dir/org.eclipse.jetty.jetty-servlet-7.6.15.v20140411.jar:$lib_dir/org.eclipse.jetty.jetty-security-7.6.15.v20140411.jar:$lib_dir/org.eclipse.jetty.jetty-server-7.6.15.v20140411.jar:$lib_dir/org.eclipse.jetty.jetty-continuation-7.6.15.v20140411.jar:$lib_dir/org.eclipse.jetty.jetty-http-7.6.15.v20140411.jar:$lib_dir/org.eclipse.jetty.jetty-io-7.6.15.v20140411.jar:$lib_dir/org.tachyonproject.tachyon-minicluster-0.7.1.jar:$lib_dir/org.apache.curator.curator-test-2.1.0-incubating.jar:$lib_dir/org.apache.commons.commons-math-2.2.jar:$lib_dir/com.typesafe.play.play_2.10-2.3.7.jar:$lib_dir/com.typesafe.play.build-link-2.3.7.jar:$lib_dir/com.typesafe.play.play-exceptions-2.3.7.jar:$lib_dir/org.javassist.javassist-3.18.2-GA.jar:$lib_dir/org.scala-stm.scala-stm_2.10-0.7.jar:$lib_dir/com.typesafe.config-1.2.1.jar:$lib_dir/org.joda.joda-convert-1.6.jar:$lib_dir/com.typesafe.play.twirl-api_2.10-1.0.2.jar:$lib_dir/io.netty.netty-3.9.3.Final.jar:$lib_dir/com.typesafe.netty.netty-http-pipelining-1.1.2.jar:$lib_dir/ch.qos.logback.logback-core-1.1.1.jar:$lib_dir/ch.qos.logback.logback-classic-1.1.1.jar:$lib_dir/commons-codec.commons-codec-1.10.jar:$lib_dir/xerces.xercesImpl-2.11.0.jar:$lib_dir/xml-apis.xml-apis-1.4.01.jar:$lib_dir/javax.transaction.jta-1.1.jar:$lib_dir/com.typesafe.akka.akka-actor_2.10-2.3.11.jar:$lib_dir/com.typesafe.akka.akka-remote_2.10-2.3.11.jar:$lib_dir/com.google.protobuf.protobuf-java-2.5.0.jar:$lib_dir/org.uncommons.maths.uncommons-maths-1.2.2a.jar:$lib_dir/com.typesafe.akka.akka-slf4j_2.10-2.3.11.jar:$lib_dir/org.apache.commons.commons-exec-1.3.jar:$lib_dir/com.github.fommil.netlib.core-1.1.2.jar:$lib_dir/net.sourceforge.f2j.arpack_combined_all-0.1.jar:$lib_dir/net.sourceforge.f2j.arpack_combined_all-0.1-javadoc.jar:$lib_dir/net.sf.opencsv.opencsv-2.3.jar:$lib_dir/com.github.rwl.jtransforms-2.4.0.jar:$lib_dir/org.spire-math.spire_2.10-0.7.4.jar:$lib_dir/org.spire-math.spire-macros_2.10-0.7.4.jar:$lib_dir/org.apache.spark.spark-core_2.10-1.5.2.jar:$lib_dir/org.apache.avro.avro-mapred-1.7.7-hadoop2.jar:$lib_dir/org.apache.avro.avro-ipc-1.7.7-tests.jar:$lib_dir/org.apache.avro.avro-ipc-1.7.7.jar:$lib_dir/org.apache.avro.avro-1.7.7.jar:$lib_dir/org.codehaus.jackson.jackson-core-asl-1.9.13.jar:$lib_dir/org.codehaus.jackson.jackson-mapper-asl-1.9.13.jar:$lib_dir/org.apache.commons.commons-compress-1.4.1.jar:$lib_dir/org.tukaani.xz-1.0.jar:$lib_dir/com.twitter.chill_2.10-0.5.0.jar:$lib_dir/com.twitter.chill-java-0.5.0.jar:$lib_dir/com.esotericsoftware.kryo.kryo-2.21.jar:$lib_dir/com.esotericsoftware.reflectasm.reflectasm-1.07-shaded.jar:$lib_dir/com.esotericsoftware.minlog.minlog-1.2.jar:$lib_dir/org.objenesis.objenesis-1.2.jar:$lib_dir/org.apache.spark.spark-launcher_2.10-1.5.2.jar:$lib_dir/org.spark-project.spark.unused-1.0.0.jar:$lib_dir/org.apache.spark.spark-network-common_2.10-1.5.2.jar:$lib_dir/io.netty.netty-all-4.0.29.Final.jar:$lib_dir/org.apache.spark.spark-network-shuffle_2.10-1.5.2.jar:$lib_dir/org.apache.spark.spark-unsafe_2.10-1.5.2.jar:$lib_dir/net.java.dev.jets3t.jets3t-0.9.0.jar:$lib_dir/org.eclipse.jetty.orbit.javax.servlet-3.0.0.v201112011016.jar:$lib_dir/org.apache.commons.commons-lang3-3.3.2.jar:$lib_dir/org.apache.commons.commons-math3-3.4.1.jar:$lib_dir/org.slf4j.slf4j-api-1.7.10.jar:$lib_dir/org.slf4j.jul-to-slf4j-1.7.10.jar:$lib_dir/org.slf4j.jcl-over-slf4j-1.7.10.jar:$lib_dir/org.slf4j.slf4j-log4j12-1.7.10.jar:$lib_dir/com.ning.compress-lzf-1.0.3.jar:$lib_dir/org.xerial.snappy.snappy-java-1.1.1.7.jar:$lib_dir/net.jpountz.lz4.lz4-1.3.0.jar:$lib_dir/org.roaringbitmap.RoaringBitmap-0.4.5.jar:$lib_dir/org.json4s.json4s-jackson_2.10-3.2.10.jar:$lib_dir/org.json4s.json4s-core_2.10-3.2.10.jar:$lib_dir/org.json4s.json4s-ast_2.10-3.2.10.jar:$lib_dir/com.thoughtworks.paranamer.paranamer-2.6.jar:$lib_dir/org.scala-lang.scalap-2.10.0.jar:$lib_dir/com.fasterxml.jackson.core.jackson-databind-2.4.4.jar:$lib_dir/com.fasterxml.jackson.core.jackson-core-2.4.4.jar:$lib_dir/com.sun.jersey.jersey-server-1.9.jar:$lib_dir/com.sun.jersey.jersey-core-1.9.jar:$lib_dir/org.apache.mesos.mesos-0.21.1-shaded-protobuf.jar:$lib_dir/com.clearspring.analytics.stream-2.7.0.jar:$lib_dir/io.dropwizard.metrics.metrics-core-3.1.2.jar:$lib_dir/io.dropwizard.metrics.metrics-jvm-3.1.2.jar:$lib_dir/io.dropwizard.metrics.metrics-json-3.1.2.jar:$lib_dir/io.dropwizard.metrics.metrics-graphite-3.1.2.jar:$lib_dir/com.fasterxml.jackson.module.jackson-module-scala_2.10-2.4.4.jar:$lib_dir/com.fasterxml.jackson.core.jackson-annotations-2.4.4.jar:$lib_dir/oro.oro-2.0.8.jar:$lib_dir/net.razorvine.pyrolite-4.4.jar:$lib_dir/net.sf.py4j.py4j-0.8.2.1.jar:$lib_dir/com.jamesmurty.utils.java-xmlbuilder-0.4.jar:$lib_dir/org.apache.spark.spark-yarn_2.10-1.5.2.jar:$lib_dir/org.apache.spark.spark-sql_2.10-1.5.2.jar:$lib_dir/org.apache.spark.spark-catalyst_2.10-1.5.2.jar:$lib_dir/org.codehaus.janino.janino-2.7.8.jar:$lib_dir/org.codehaus.janino.commons-compiler-2.7.8.jar:$lib_dir/org.apache.parquet.parquet-column-1.7.0.jar:$lib_dir/org.apache.parquet.parquet-common-1.7.0.jar:$lib_dir/org.apache.parquet.parquet-encoding-1.7.0.jar:$lib_dir/org.apache.parquet.parquet-generator-1.7.0.jar:$lib_dir/org.apache.parquet.parquet-hadoop-1.7.0.jar:$lib_dir/org.apache.parquet.parquet-format-2.3.0-incubating.jar:$lib_dir/org.apache.parquet.parquet-jackson-1.7.0.jar:$lib_dir/org.apache.hadoop.hadoop-client-2.7.1.jar:$lib_dir/org.apache.hadoop.hadoop-common-2.7.1.jar:$lib_dir/org.apache.hadoop.hadoop-annotations-2.7.1.jar:$lib_dir/commons-cli.commons-cli-1.2.jar:$lib_dir/xmlenc.xmlenc-0.52.jar:$lib_dir/commons-httpclient.commons-httpclient-3.1.jar:$lib_dir/commons-logging.commons-logging-1.1.3.jar:$lib_dir/commons-net.commons-net-3.1.jar:$lib_dir/commons-collections.commons-collections-3.2.1.jar:$lib_dir/commons-lang.commons-lang-2.6.jar:$lib_dir/commons-configuration.commons-configuration-1.6.jar:$lib_dir/commons-digester.commons-digester-1.8.jar:$lib_dir/commons-beanutils.commons-beanutils-1.7.0.jar:$lib_dir/commons-beanutils.commons-beanutils-core-1.8.0.jar:$lib_dir/com.google.code.gson.gson-2.2.4.jar:$lib_dir/org.apache.hadoop.hadoop-auth-2.7.1.jar:$lib_dir/org.apache.directory.server.apacheds-kerberos-codec-2.0.0-M15.jar:$lib_dir/org.apache.directory.server.apacheds-i18n-2.0.0-M15.jar:$lib_dir/org.apache.directory.api.api-asn1-api-1.0.0-M20.jar:$lib_dir/org.apache.directory.api.api-util-1.0.0-M20.jar:$lib_dir/org.apache.zookeeper.zookeeper-3.4.6.jar:$lib_dir/org.apache.curator.curator-framework-2.7.1.jar:$lib_dir/org.apache.curator.curator-client-2.7.1.jar:$lib_dir/org.apache.curator.curator-recipes-2.7.1.jar:$lib_dir/com.google.code.findbugs.jsr305-3.0.0.jar:$lib_dir/org.apache.htrace.htrace-core-3.1.0-incubating.jar:$lib_dir/org.apache.hadoop.hadoop-hdfs-2.7.1.jar:$lib_dir/org.mortbay.jetty.jetty-util-6.1.26.jar:$lib_dir/org.fusesource.leveldbjni.leveldbjni-all-1.8.jar:$lib_dir/org.apache.hadoop.hadoop-mapreduce-client-app-2.7.1.jar:$lib_dir/org.apache.hadoop.hadoop-mapreduce-client-common-2.7.1.jar:$lib_dir/org.apache.hadoop.hadoop-yarn-common-2.7.1.jar:$lib_dir/org.apache.hadoop.hadoop-yarn-api-2.7.1.jar:$lib_dir/com.sun.jersey.jersey-client-1.9.jar:$lib_dir/org.codehaus.jackson.jackson-jaxrs-1.9.13.jar:$lib_dir/org.codehaus.jackson.jackson-xc-1.9.13.jar:$lib_dir/com.google.inject.guice-3.0.jar:$lib_dir/javax.inject.javax.inject-1.jar:$lib_dir/aopalliance.aopalliance-1.0.jar:$lib_dir/org.sonatype.sisu.inject.cglib-2.2.1-v20090111.jar:$lib_dir/asm.asm-3.2.jar:$lib_dir/com.sun.jersey.jersey-json-1.9.jar:$lib_dir/org.codehaus.jettison.jettison-1.1.jar:$lib_dir/com.sun.jersey.contribs.jersey-guice-1.9.jar:$lib_dir/org.apache.hadoop.hadoop-yarn-client-2.7.1.jar:$lib_dir/org.apache.hadoop.hadoop-mapreduce-client-core-2.7.1.jar:$lib_dir/org.apache.hadoop.hadoop-yarn-server-common-2.7.1.jar:$lib_dir/org.apache.hadoop.hadoop-mapreduce-client-shuffle-2.7.1.jar:$lib_dir/org.apache.hadoop.hadoop-mapreduce-client-jobclient-2.7.1.jar:$lib_dir/javax.servlet.jsp.jsp-api-2.1.jar:$lib_dir/com.databricks.spark-csv_2.10-1.3.0.jar:$lib_dir/org.apache.commons.commons-csv-1.1.jar:$lib_dir/com.univocity.univocity-parsers-1.5.1.jar:$lib_dir/org.apache.spark.spark-repl_2.10-1.5.2.jar:$lib_dir/org.scala-lang.jline-2.10.4.jar:$lib_dir/org.fusesource.jansi.jansi-1.4.jar:$lib_dir/org.apache.spark.spark-bagel_2.10-1.5.2.jar:$lib_dir/org.apache.spark.spark-mllib_2.10-1.5.2.jar:$lib_dir/org.apache.spark.spark-streaming_2.10-1.5.2.jar:$lib_dir/org.apache.spark.spark-graphx_2.10-1.5.2.jar:$lib_dir/org.scalanlp.breeze_2.10-0.11.2.jar:$lib_dir/org.scalanlp.breeze-macros_2.10-0.11.2.jar:$lib_dir/org.jpmml.pmml-model-1.1.15.jar:$lib_dir/org.jpmml.pmml-agent-1.1.15.jar:$lib_dir/org.jpmml.pmml-schema-1.1.15.jar:$lib_dir/com.sun.xml.bind.jaxb-impl-2.2.7.jar:$lib_dir/com.sun.xml.bind.jaxb-core-2.2.7.jar:$lib_dir/javax.xml.bind.jaxb-api-2.2.7.jar:$lib_dir/com.sun.istack.istack-commons-runtime-2.16.jar:$lib_dir/com.sun.xml.fastinfoset.FastInfoset-1.2.12.jar:$lib_dir/javax.xml.bind.jsr173_api-1.0.jar:$lib_dir/org.apache.spark.spark-hive_2.10-1.5.2.jar:$lib_dir/com.twitter.parquet-hadoop-bundle-1.6.0.jar:$lib_dir/org.spark-project.hive.hive-exec-1.2.1.spark.jar:$lib_dir/javolution.javolution-5.5.1.jar:$lib_dir/log4j.apache-log4j-extras-1.2.17.jar:$lib_dir/org.antlr.antlr-runtime-3.4.jar:$lib_dir/org.antlr.stringtemplate-3.2.1.jar:$lib_dir/antlr.antlr-2.7.7.jar:$lib_dir/org.antlr.ST4-4.0.4.jar:$lib_dir/org.codehaus.groovy.groovy-all-2.1.6.jar:$lib_dir/org.jodd.jodd-core-3.5.2.jar:$lib_dir/org.datanucleus.datanucleus-core-3.2.10.jar:$lib_dir/org.apache.calcite.calcite-avatica-1.2.0-incubating.jar:$lib_dir/com.googlecode.javaewah.JavaEWAH-0.3.2.jar:$lib_dir/org.iq80.snappy.snappy-0.2.jar:$lib_dir/org.json.json-20090211.jar:$lib_dir/stax.stax-api-1.0.1.jar:$lib_dir/jline.jline-2.12.jar:$lib_dir/org.spark-project.hive.hive-metastore-1.2.1.spark.jar:$lib_dir/com.jolbox.bonecp-0.8.0.RELEASE.jar:$lib_dir/org.apache.derby.derby-10.10.2.0.jar:$lib_dir/org.datanucleus.datanucleus-api-jdo-3.2.6.jar:$lib_dir/org.datanucleus.datanucleus-rdbms-3.2.9.jar:$lib_dir/commons-pool.commons-pool-1.5.4.jar:$lib_dir/commons-dbcp.commons-dbcp-1.4.jar:$lib_dir/javax.jdo.jdo-api-3.0.1.jar:$lib_dir/org.apache.calcite.calcite-core-1.2.0-incubating.jar:$lib_dir/org.apache.calcite.calcite-linq4j-1.2.0-incubating.jar:$lib_dir/net.hydromatic.eigenbase-properties-1.1.5.jar:$lib_dir/org.apache.httpcomponents.httpclient-4.3.2.jar:$lib_dir/org.apache.httpcomponents.httpcore-4.3.1.jar:$lib_dir/org.apache.thrift.libthrift-0.9.2.jar:$lib_dir/org.apache.thrift.libfb303-0.9.2.jar:$lib_dir/org.apache.hadoop.hadoop-yarn-server-web-proxy-2.7.1.jar:$lib_dir/org.mortbay.jetty.jetty-6.1.26.jar:$lib_dir/io.reactivex.rxscala_2.10-0.22.0.jar:$lib_dir/io.reactivex.rxjava-1.0.0-rc.5.jar:$lib_dir/org.scalaz.scalaz-core_2.10-7.0.6.jar:$lib_dir/org.scala-sbt.sbt-0.13.8.jar:$lib_dir/org.scala-sbt.main-0.13.8.jar:$lib_dir/org.scala-sbt.actions-0.13.8.jar:$lib_dir/org.scala-sbt.classpath-0.13.8.jar:$lib_dir/org.scala-sbt.launcher-interface-0.13.8.jar:$lib_dir/org.scala-sbt.interface-0.13.8.jar:$lib_dir/org.scala-sbt.io-0.13.8.jar:$lib_dir/org.scala-sbt.control-0.13.8.jar:$lib_dir/org.scala-sbt.completion-0.13.8.jar:$lib_dir/org.scala-sbt.collections-0.13.8.jar:$lib_dir/org.scala-sbt.api-0.13.8.jar:$lib_dir/org.scala-sbt.compiler-integration-0.13.8.jar:$lib_dir/org.scala-sbt.incremental-compiler-0.13.8.jar:$lib_dir/org.scala-sbt.logging-0.13.8.jar:$lib_dir/org.scala-sbt.process-0.13.8.jar:$lib_dir/org.scala-sbt.relation-0.13.8.jar:$lib_dir/org.scala-sbt.compile-0.13.8.jar:$lib_dir/org.scala-sbt.classfile-0.13.8.jar:$lib_dir/org.scala-sbt.persist-0.13.8.jar:$lib_dir/org.scala-tools.sbinary.sbinary_2.10-0.4.2.jar:$lib_dir/org.scala-sbt.compiler-ivy-integration-0.13.8.jar:$lib_dir/org.scala-sbt.ivy-0.13.8.jar:$lib_dir/org.scala-sbt.cross-0.13.8.jar:$lib_dir/org.scala-sbt.ivy.ivy-2.3.0-sbt-fccfbd44c9f64523b61398a0155784dcbaeae28f.jar:$lib_dir/com.jcraft.jsch-0.1.46.jar:$lib_dir/org.scala-sbt.serialization_2.10-0.1.1.jar:$lib_dir/org.scala-lang.modules.scala-pickling_2.10-0.10.0.jar:$lib_dir/org.scalamacros.quasiquotes_2.10-2.0.1.jar:$lib_dir/org.spire-math.jawn-parser_2.10-0.6.0.jar:$lib_dir/org.spire-math.json4s-support_2.10-0.6.0.jar:$lib_dir/org.scala-sbt.run-0.13.8.jar:$lib_dir/org.scala-sbt.task-system-0.13.8.jar:$lib_dir/org.scala-sbt.tasks-0.13.8.jar:$lib_dir/org.scala-sbt.tracking-0.13.8.jar:$lib_dir/org.scala-sbt.cache-0.13.8.jar:$lib_dir/org.scala-sbt.testing-0.13.8.jar:$lib_dir/org.scala-sbt.test-agent-0.13.8.jar:$lib_dir/org.scala-sbt.test-interface-1.0.jar:$lib_dir/org.scala-sbt.main-settings-0.13.8.jar:$lib_dir/org.scala-sbt.apply-macro-0.13.8.jar:$lib_dir/org.scala-sbt.command-0.13.8.jar:$lib_dir/org.scala-sbt.logic-0.13.8.jar:$lib_dir/org.scala-sbt.compiler-interface--bin-0.13.8.jar:$lib_dir/org.scala-sbt.compiler-interface--src-0.13.8.jar:$lib_dir/org.scala-sbt.precompiled-2_8_2-compiler-interface-bin-0.13.8.jar:$lib_dir/org.scala-sbt.precompiled-2_9_2-compiler-interface-bin-0.13.8.jar:$lib_dir/org.scala-sbt.precompiled-2_9_3-compiler-interface-bin-0.13.8.jar:$lib_dir/com.frugalmechanic.fm-sbt-s3-resolver-0.5.0.jar:$lib_dir/com.amazonaws.aws-java-sdk-s3-1.9.0.jar:$lib_dir/com.amazonaws.aws-java-sdk-core-1.9.0.jar:$lib_dir/joda-time.joda-time-2.9.1.jar:$lib_dir/io.continuum.bokeh.bokeh_2.10-0.2.jar:$lib_dir/io.continuum.bokeh.core_2.10-0.2.jar:$lib_dir/com.typesafe.play.play-json_2.10-2.4.0-M1.jar:$lib_dir/com.typesafe.play.play-iteratees_2.10-2.4.0-M1.jar:$lib_dir/com.typesafe.play.play-functional_2.10-2.4.0-M1.jar:$lib_dir/com.typesafe.play.play-datacommons_2.10-2.4.0-M1.jar:$lib_dir/io.continuum.bokeh.bokehjs_2.10-0.2.jar:$lib_dir/com.github.scala-incubator.io.scala-io-core_2.10-0.4.3.jar:$lib_dir/com.jsuereth.scala-arm_2.10-1.3.jar:$lib_dir/com.github.scala-incubator.io.scala-io-file_2.10-0.4.3.jar:$lib_dir/com.quantifind.sumac_2.10-0.3.0.jar:$lib_dir/org.wololo.jts2geojson-0.7.0.jar:$lib_dir/com.vividsolutions.jts-1.13.jar:$lib_dir/com.typesafe.play.play-cache_2.10-2.3.7.jar:$lib_dir/net.sf.ehcache.ehcache-core-2.6.8.jar:$lib_dir/nooostab.spark-notebook-0.6.2-scala-2.10.4-spark-1.5.2-hadoop-2.7.1-with-hive-with-parquet-assets.jar"

addJava "-Duser.dir=$(cd "${app_home}/.."; pwd -P)"
addJava "-Dhdp.version=2.4.0.0-169 -Dspark.yarn.queue=default"

export ADD_JARS="${ADD_JARS},${lib_dir}/$(ls ${lib_dir} | grep common.common | head)"
# java_cmd is overrode in process_args when -java-home is used
declare java_cmd=$(get_java_cmd)

# if configuration files exist, prepend their contents to $@ so it can be processed by this runner
[[ -f "$script_conf_file" ]] && set -- $(loadConfigFile "$script_conf_file") "$@"
#source cluster 
export MASTER=yarn-client
. /etc/spark/conf/spark-env.sh
export PYSPARK_SUBMIT_ARGS="--packages com.databricks:spark-csv_2.10:1.4.0 --master yarn-client pyspark-shell"

# Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode
export MASTER=yarn-client
export SPARK_YARN_JAR=hdfs://cn100.c3.com:8020/hdp/apps/2.4.0.0-169/spark/spark-hdp-assembly.jar
# Additional jvm options. for example, export ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16"
export JAVA_OPTS="-Dhdp.version=2.4.0.0-169 -Dspark.executor.memory=1024m -Dspark.executor.instances=2 -Dspark.yarn.queue=default"
##
# (required) When it is defined, load it instead of Zeppelin embedded Spark libraries
export SPARK_HOME=/usr/hdp/current/spark-client/

export PYTHONPATH="${SPARK_HOME}/python:${SPARK_HOME}/python/lib/py4j-0.8.2.1-src.zip"
export SPARK_YARN_USER_ENV="PYTHONPATH=${PYTHONPATH}"


run "$@"

xadmin@c3gw:/usr/share/spark-notebook/bin$ ps -edf |grep spark-notebook
root 3342 1 0 08:48 ? 00:00:00 sudo -u spark-notebook bin/spark-notebook
spark-n+ 3344 3342 8 08:48 ? 00:00:03 java -Dhdp.version=2.4.0.0-169 -Dspark.executor.memory=1024m -Dspark.executor.instances=2 -Dspark.yarn.queue=default -Duser.dir=/usr/share/spark-notebook -D-Dhdp.version=2.4.0.0-169 -Dspark.yarn.queue=default -cp ::::/usr/share/spark-notebook/lib/nooostab.spark-notebook-0.6.2-scala-2.10.4-spark-1.5.2-hadoop-2.7.1-with-hive-with-parquet.jar:/usr/share/spark-notebook/lib/tachyon.tachyon-0.6.2-scala-2.10.4-spark-1.5.2-hadoop-2.7.1-with-hive-with-parquet.jar:/usr/share/spark-notebook/lib/subprocess.subprocess-0.6.2-scala-2.10.4-spark-1.5.2-hadoop-2.7.1-with-hive-with-parquet.jar:/usr/share/spark-notebook/lib/observable.observable-0.6.2-scala-2.10.4-spark-1.5.2-hadoop-2.7.1-with-hive-with-parquet.jar:/usr/share/spark-notebook/lib/common.common-0.6.2-scala-2.10.4-spark-1.5.2-hadoop-2.7.1-with-hive-with-parquet.jar:/usr/share/spark-notebook/lib/spark.spark-0.6.2-scala-2.10.4-spark-1.5.2-hadoop-2.7.1-with-hive-with-parquet.jar:/usr/share/spark-notebook/lib/kernel.kernel-0.6.2-scala-2.10.4-spark-1.5.2-hadoop-2.7.1-with-hive-with-parquet.jar:/usr/share/spark-notebook/lib/wisp_2.10-0.0.5.jar:/usr/share/spark-notebook/lib/org.scala-lang.scala-compiler-2.10.4.jar:/usr/share/spark-notebook/lib/org.scala-lang.scala-library-2.10.4.jar:/usr/share/spark-notebook/lib/org.scala-lang.scala-reflect-2.10.4.jar:/usr/share/spark-notebook/lib/com.google.guava.guava-14.0.1.jar:/usr/share/spark-notebook/lib/org.tachyonproject.tachyon-common-0.7.1.jar:/usr/share/spark-notebook/lib/commons-io.commons-io-2.4.jar:/usr/share/spark-notebook/lib/log4j.log4j-1.2.16.jar:/usr/share/spark-notebook/lib/org.tachyonproject.tachyon-client-0.7.1.jar:/usr/share/spark-notebook/lib/org.tachyonproject.tachyon-underfs-hdfs-0.7.1.jar:/usr/share/spark-notebook/lib/org.tachyonproject.tachyon-underfs-local-0.7.1.jar:/usr/share/spark-notebook/lib/org.tachyonproject.tachyon-servers-0.7.1.jar:/usr/share/spark-notebook/lib/org.tachyonproject.tachyon-client-unshaded-0.7.1.jar:/usr/share/spark-notebook/lib/org.eclipse.jetty.jetty-jsp-7.6.15.v20140411.jar:/usr/share/spark-notebook/lib/org.eclipse.jetty.orbit.javax.servlet.jsp-2.1.0.v201105211820.jar:/usr/share/spark-notebook/lib/org.eclipse.jetty.orbit.org.apache.jasper.glassfish-2.1.0.v201110031002.jar:/usr/share/spark-notebook/lib/org.eclipse.jetty.orbit.javax.servlet.jsp.jstl-1.2.0.v201105211821.jar:/usr/share/spark-notebook/lib/org.eclipse.jetty.orbit.org.apache.taglibs.standard.glassfish-1.2.0.v201112081803.jar:/usr/share/spark-notebook/lib/org.eclipse.jetty.orbit.javax.el-2.1.0.v201105211819.jar:/usr/share/spark-notebook/lib/org.eclipse.jetty.orbit.com.sun.el-1.0.0.v201105211818.jar:/usr/share/spark-notebook/lib/org.eclipse.jetty.orbit.org.eclipse.jdt.core-3.7.1.jar:/usr/share/spark-notebook/lib/org.eclipse.jetty.jetty-webapp-7.6.15.v20140411.jar:/usr/share/spark-notebook/lib/org.eclipse.jetty.jetty-xml-7.6.15.v20140411.jar:/usr/share/spark-notebook/lib/org.eclipse.jetty.jetty-util-7.6.15.v20140411.jar:/usr/share/spark-notebook/lib/org.eclipse.jetty.jetty-servlet-7.6.15.v20140411.jar:/usr/share/spark-notebook/lib/org.eclipse.jetty.jetty-security-7.6.15.v20140411.jar:/usr/share/spark-notebook/lib/org.eclipse.jetty.jetty-server-7.6.15.v20140411.jar:/usr/share/spark-notebook/lib/org.eclipse.jetty.jetty-continuation-7.6.15.v20140411.jar:/usr/share/spark-notebook/lib/org.eclipse.jetty.jetty-http-7.6.15.v20140411.jar:/usr/share/spark-notebook/lib/org.eclipse.jetty.jetty-io-7.6.15.v20140411.jar:/usr/share/spark-notebook/lib/org.tachyonproject.tachyon-minicluster-0.7.1.jar:/usr/share/spark-notebook/lib/org.apache.curator.curator-test-2.1.0-incubating.jar:/usr/share/spark-notebook/lib/org.apache.commons.commons-math-2.2.jar:/usr/share/spark-notebook/lib/com.typesafe.play.play_2.10-2.3.7.jar:/usr/share/spark-notebook/lib/com.typesafe.play.build-link-2.3.7.jar:/usr/share/spark-notebook/lib/com.typesafe.play.play-exceptions-2.3.7.jar:/usr/share/spark-notebook/lib/org.javassist.javassist-3.18.2-GA.jar:/usr/share/spark-notebook/lib/org.scala-stm.scala-stm_2.10-0.7.jar:/usr/share/spark-notebook/lib/com.typesafe.config-1.2.1.jar:/usr/share