run-spark-pi-3.3-SNAPSHOT-on-k8s.bash

#!/usr/bin/env bash

DRIVER_CORES=1
DRIVER_MEM=2048m
EXECUTOR_INSTANCES=3
EXECUTOR_CORES=2
EXECUTOR_MEM=2048m
SPARK_HOME="/path/to/spark-3.3.0-SNAPSHOT-bin-with-volcano" # CHANGE 1

PATH="$JAVA_8_HOME/bin:$SPARK_HOME/bin:$PATH"

CURRENT_CONTEXT=$(kubectl config view -o jsonpath="{.current-context}")

K8S_MASTER=$(kubectl config view -o jsonpath="{.clusters[?(@.name==\"$CURRENT_CONTEXT\")].cluster.server}") 

CONTAINER_IMAGE="spark:3.3.0-SNAPSHOT-scala_2.12-11-jre-slim"   # CHANGE 2

echo "K8S_MASTER: $K8S_MASTER"

spark-submit \
           --verbose \
           --master k8s://$K8S_MASTER \
           --deploy-mode cluster \
           --name {$K8S_NAMESPACE}-app \
           --conf spark.kubernetes.namespace=$K8S_NAMESPACE \
           --conf spark.network.timeout=300 \
           --conf spark.executor.instances=$EXECUTOR_INSTANCES \
           --conf spark.driver.cores=$DRIVER_CORES \
           --conf spark.executor.cores=$EXECUTOR_CORES \
           --conf spark.driver.memory=$DRIVER_MEM \
           --conf spark.executor.memory=$EXECUTOR_MEM \
           --conf spark.kubernetes.driver.container.image=$CONTAINER_IMAGE \
           --conf spark.kubernetes.executor.container.image=$CONTAINER_IMAGE \
           --conf spark.kubernetes.authenticate.driver.serviceAccountName=$K8S_SERVICE_ACCOUNT_NAME \
           --conf spark.driver.extraJavaOptions="-Dio.netty.tryReflectionSetAccessible=true" \
           --conf spark.executor.extraJavaOptions="-Dio.netty.tryReflectionSetAccessible=true" \
           --class org.apache.spark.examples.SparkPi \
          local:///opt/spark/examples/jars/spark-examples_2.12-3.3.0-SNAPSHOT.jar   # CHANGE 3