medium-spark-shell-on-k8s.sh

##############
### Prepare for env variables
export NS_NAME=spark-ext
export SA_NAME=spark
export CLN_NAME=spark-client
export POD_IMAG=gcr.io/spark-operator/spark-operator:v1beta2-1.2.3-3.1.1
export SVC_NAME=$CLN_NAME-headless
export SVC_PORT=19987

export CLS_ENDP="https://c2.us-south.containers.cloud.ibm.com:26165"

export EXR_INST=4
export EXR_MORY=10g
export DRV_MORY=8g


##############
### Create all resources 
# NS_NAME
kubectl create ns $NS_NAME \
  --dry-run=client -o yaml | kubectl apply -f -
kubectl config set-context --current --namespace=$NS_NAME

# SA_NAME
kubectl create sa $SA_NAME \
  --dry-run=client -o yaml | kubectl apply -f -
kubectl create clusterrolebinding ${SA_NAME}-${NS_NAME}-edit \
  --clusterrole=edit \
  --serviceaccount=$NS_NAME:$SA_NAME \
  --namespace=$NS_NAME \
  --dry-run=client -o yaml | kubectl apply -f -

# CLN_NAME and POD_IMAG
kubectl run $CLN_NAME \
  --image=$POD_IMAG \
  --image-pull-policy=Always \
  --serviceaccount=$SA_NAME \
  --dry-run=client -o yaml \
  --command=true -- sh -c "exec tail -f /dev/null" | kubectl apply -f -
  
# SVC_NAME and SVC_PORT
kubectl expose pod $CLN_NAME \
  --name=$SVC_NAME \
  --type=ClusterIP \
  --cluster-ip=None \
  --port=$SVC_PORT \
  --dry-run=client -o yaml | kubectl apply -f -
  

##############
### Start spark-shell 
kubectl exec -it $CLN_NAME -- sh -c '\
cd /opt/spark/; \
./bin/spark-shell  \
--master k8s://"'$CLS_ENDP'" \
--deploy-mode client  \
--conf spark.kubernetes.namespace="'$NS_NAME'"  \
--conf spark.kubernetes.container.image="'$POD_IMAG'" \
--conf spark.kubernetes.container.image.pullPolicy=Always  \
--conf spark.kubernetes.authenticate.serviceAccountName="'$SA_NAME'"  \
--conf spark.kubernetes.driver.pod.name="'$CLN_NAME'"  \
--conf spark.executor.instances="'$EXR_INST'"  \
--conf spark.executor.memory="'$EXR_MORY'" \
--conf spark.driver.memory="'$DRV_MORY'" \
--conf spark.driver.host="'$SVC_NAME'" \
--conf spark.driver.port="'$SVC_PORT'"  \
--conf spark.jars.ivy=/tmp/.ivy'