Deploy an on-premise data hub with Canonical MAAS, Spark, Kubernetes and Ceph

charmed-spark-datahub.sh

#!/bin/bash

# 
# author:        Rob Gibbon, Canonical Ltd.
# about:         This script deploys a complete, charmed data lake stack on MAAS (https://maas.io)
# prerequisites: * A host computer running Ubuntu
#                * A working MAAS environment, with provisioned 
#                  nodes available and configured for KVM/libvirt
#                * A Kaggle account and valid API token configured
#                * Google Chrome browser installation for WUIs

#######################################
#     
#      Variables
#
#######################################

MAAS_IP=192.168.86.45
MAAS_URL=https://${MAAS_IP}:5443/MAAS
MAAS_USER=rob
MAAS_TOKEN=yourtoken
METALLB_RANGE=192.168.86.90-192.168.86.95
MICROK8S_NODE_COUNT=3
MICROK8S_NODE_MEM_GB=8
MICROK8S_NODE_DISK_GB=20
CEPH_NODE_OSD_DISK_GB=1
CEPH_NODE_OSD_DISK_COUNT=1
CEPH_NODE_OSD_COUNT=3
CEPH_NODE_MON_COUNT=3
VAULT_NODE_MEM_GB=1

#######################################
#
#      Files
#
#######################################

if [ ! -f ${HOME}/.kaggle/kaggle.json ]; then
  echo "You first need to set up your Kaggle API token. Go to https://www.kaggle.com/ and create an API token or sign up"
  exit -1
fi

cat > pyspark-script.py <<EOF
df = spark.read.option("header", "true").csv("s3a://data/traffic-collision-data-from-2010-to-present.csv")
df.createOrReplaceTempView("collisions")
spark.sql("select \`DR Number\` from collisions group by \`DR Number\` having count(\`DR Number\`) > 1").show()
quit()
EOF

cat > maas-credential.yaml <<EOF
credentials:
  maas:
    ${MAAS_USER}:
      auth-type: oauth1
      maas-oauth: ${MAAS_TOKEN}
EOF

#######################################
#
#      Prerequisites
#
#######################################

sudo snap remove --purge juju
sudo snap remove --purge juju-wait
sudo snap remove --purge minio-mc-nsg
sudo snap remove --purge spark-client
sudo snap remove --purge vault

sudo snap install juju --channel=3.3/stable
sudo snap install juju-wait --channel=latest/edge --classic
sudo snap install minio-mc-nsg
sudo snap alias minio-mc-nsg mc
sudo snap install spark-client --channel=3.4/edge
sudo snap install vault

sudo apt install jq yq unzip openssl mkcert -y

pip install kaggle

#######################################
#
#      Commands
#
#######################################

# enable MAAS TLS
mkcert -install
mkcert maas.datahub.demo ${MAAS_IP}

cp ${HOME}/.local/share/mkcert/rootCA.pem .
sudo cp rootCA.pem /var/snap/maas/common
sudo cp maas.datahub.demo+1.pem /var/snap/maas/common
sudo cp maas.datahub.demo+1-key.pem /var/snap/maas/common

echo "y" | sudo maas config-tls enable --port 5443 --cacert /var/snap/maas/common/rootCA.pem /var/snap/maas/common/maas.datahub.demo.key /var/snap/maas/common/maas.datahub.demo.pem

cat > maas-cloud.yaml <<EOF
clouds:
  maas:
    type: maas
    auth-types: [oauth1]
    endpoint: ${MAAS_URL}
    ca-certificates:
    - |
$(cat rootCA.pem | sed -e 's/^/      /')
EOF

cat > cloudinit-userdata.yaml <<EOF
cloudinit-userdata: |
  ca-certs:
    trusted: 
    - |
$(cat rootCA.pem | sed -e 's/^/      /')
EOF

# create the MAAS cloud
juju add-cloud --client maas -f maas-cloud.yaml
juju add-credential maas -f maas-credential.yaml 
juju bootstrap maas --credential ${MAAS_USER} --model-default cloudinit-userdata.yaml cloud-controller
juju switch cloud-controller
juju enable-ha

# create the foundations - Ceph & MicroK8s
juju add-model charm-stack-base-model maas
juju deploy microk8s -n ${MICROK8S_NODE_COUNT} --config hostpath_storage=true --constraints "mem=${MICROK8S_NODE_MEM_GB}G root-disk=${MICROK8S_NODE_DISK_GB}G" --channel=edge
juju-wait
juju deploy ceph-osd --storage osd-devices=loop,${CEPH_NODE_OSD_DISK_GB}G,${CEPH_NODE_OSD_DISK_COUNT} -n ${CEPH_NODE_OSD_COUNT}; juju-wait
juju deploy -n ${CEPH_NODE_MON_COUNT} ceph-mon; juju-wait
juju deploy ceph-radosgw; juju-wait
juju integrate ceph-radosgw:mon ceph-mon:radosgw
juju integrate ceph-osd:mon ceph-mon:osd
juju deploy grafana-agent --channel edge; juju-wait
juju integrate microk8s:cos-agent grafana-agent

# deploy Vault for Ceph TLS
juju deploy vault --constraints "mem=${VAULT_NODE_MEM_GB}G" --channel=1.8/stable; juju-wait
VAULT_IP=$(juju status | grep vault | tail -n 1 | awk '{ print $5 }')

mkcert vault.datahub.demo ${VAULT_IP}

juju config vault ssl-ca="$(cat rootCA.pem | base64)"
juju config vault ssl-cert="$(cat vault.datahub.demo+1.pem | base64)"
juju config vault ssl-key="$(cat vault.datahub.demo+1-key.pem | base64)"

juju-wait

export VAULT_ADDR="https://${VAULT_IP}:8200"
VAULT_OUTPUT=$(vault operator init -key-shares=5 -key-threshold=3)
KEY1=$(echo ${VAULT_OUTPUT} | grep "Unseal Key 1" | awk '{ print $4}')
KEY2=$(echo ${VAULT_OUTPUT} | grep "Unseal Key 2" | awk '{ print $4}')
KEY3=$(echo ${VAULT_OUTPUT} | grep "Unseal Key 3" | awk '{ print $4}')
KEY4=$(echo ${VAULT_OUTPUT} | grep "Unseal Key 4" | awk '{ print $4}')
KEY5=$(echo ${VAULT_OUTPUT} | grep "Unseal Key 5" | awk '{ print $4}')
export VAULT_TOKEN=$(echo ${VAULT_OUTPUT} | grep "Initial Root Token" | awk '{ print $4 }')

echo "Do not lose these keys!"
echo 
echo "unseal key 1: ${KEY1}"
echo "unseal key 2: ${KEY2}"
echo "unseal key 3: ${KEY3}"
echo "unseal key 4: ${KEY4}"
echo "unseal key 5: ${KEY5}"
echo
echo "root token: ${VAULT_TOKEN}"

vault operator unseal ${KEY1}
vault operator unseal ${KEY2}
vault operator unseal ${KEY3}

VAULT_JUJU_TOKEN_OUTPUT=$(vault token create -ttl=10m)
VAULT_JUJU_TOKEN=$(echo ${VAULT_JUJU_TOKEN_OUTPUT} | grep token | head -n 1 | awk '{ print $2 }')

juju run vault/leader authorize-charm token=${VAULT_JUJU_TOKEN}; juju-wait
juju run vault/leader generate-root-ca; juju-wait
juju integrate ceph-radosgw:certificates vault:certificates

juju expose ceph-radosgw
juju expose microk8s

# Import Ceph CA into local trusted CA store
CEPH_ROOT_CA_OUTPUT=$(juju run vault/leader get-root-ca)
echo ${CEPH_ROOT_CA_OUTPUT} | tail -n +2 | grep "^\s\s.*$" | sed "s/\ \ //g" > ceph-ca.pem
sudo cp ceph-ca.pem /usr/local/share/ca-certificates
sudo update-ca-certificates

# configure Ceph
# create a user account
CEPH_RESPONSE_JSON=$(juju ssh ceph-mon/leader 'sudo radosgw-admin user create --uid="ubuntu" --display-name="Charmed Spark User"')

CEPH_ACCESS_KEY_ID=$(echo ${CEPH_RESPONSE_JSON} | yq -r '.keys[].access_key')
CEPH_SECRET_ACCESS_KEY=$(echo ${CEPH_RESPONSE_JSON} | yq -r '.keys[].secret_key')

# get RadosGW IP address
CEPH_IP=$(juju status | grep ceph-radosgw | tail -n 1 | awk '{ print $5 }')
mc config host add ceph-radosgw https://${CEPH_IP} ${CEPH_ACCESS_KEY_ID} ${CEPH_SECRET_ACCESS_KEY}
mc mb ceph-radosgw/spark-history
mc mb ceph-radosgw/data

cat > policy-data-bucket.json <<EOF
{
  "Version": "2012-10-17",
  "Id": "s3policy1",
  "Statement": [{
    "Sid": "BucketAllow",
    "Effect": "Allow",
    "Principal": {"AWS": ["arn:aws:iam::user/ubuntu"]},
    "Action": [ "s3:ListBucket", "s3:PutObject", "s3:GetObject" ],
    "Resource": [
      "arn:aws:s3:::data", "arn:aws:s3:::data/*"
    ]
  }]
}
EOF

cat > policy-spark-history-bucket.json <<EOF
{
  "Version": "2012-10-17",
  "Id": "s3policy2",
  "Statement": [{
    "Sid": "BucketAllow",
    "Effect": "Allow",
    "Principal": {"AWS": ["arn:aws:iam::user/ubuntu"]},
    "Action": [ "s3:ListBucket", "s3:PutObject", "s3:GetObject" ],
    "Resource": [
      "arn:aws:s3:::spark-history", "arn:aws:s3:::spark-history/*"
    ]
  }]
}
EOF

mc policy set-json ./policy1.json ceph-radosgw/data
mc policy set-json ./policy1.json ceph-radosgw/spark-history

# scult the Kubernetes layer
KUBECONF="$(juju exec --unit microk8s/leader -- microk8s config)"
echo "${KUBECONF}" | juju add-k8s microk8s-cloud --controller cloud-controller
juju add-model spark-model microk8s-cloud

# metallb charm
juju add-model metallb-system microk8s-cloud
juju deploy metallb --channel 1.29/beta --trust; juju-wait
juju config metallb iprange="${METALLB_RANGE}"

# Spark history
juju switch spark-model
juju deploy spark-history-server-k8s
juju deploy s3-integrator --channel=latest/edge
juju deploy traefik-k8s --trust
juju-wait
juju config s3-integrator bucket="spark-history" path="spark-events" endpoint=https://${CEPH_IP} tls-ca-chain="$(cat ceph-ca.pem)"
juju run s3-integrator/leader sync-s3-credentials access-key=${CEPH_ACCESS_KEY_ID} secret-key=${CEPH_SECRET_ACCESS_KEY}
juju integrate s3-integrator spark-history-server-k8s
juju integrate traefik-k8s spark-history-server-k8s

# set up COS
juju add-model cos-model microk8s-cloud

curl -L https://raw.githubusercontent.com/canonical/cos-lite-bundle/main/overlays/offers-overlay.yaml -O
curl -L https://raw.githubusercontent.com/canonical/cos-lite-bundle/main/overlays/storage-small-overlay.yaml -O

juju deploy cos-lite \
  --trust \
  --overlay ./offers-overlay.yaml \
  --overlay ./storage-small-overlay.yaml

juju deploy cos-configuration-k8s --config git_repo=https://github.com/canonical/charmed-spark-rock --config git_branch=dashboard \
  --config git_depth=1 --config grafana_dashboards_path=dashboards/prod/grafana/
juju-wait

juju integrate cos-configuration-k8s grafana

juju offer prometheus:receive-remote-write prometheus
juju offer loki:logging loki
juju offer grafana:grafana-dashboard grafana

# COS - Microk8s integration
juju switch charm-stack-base-model 
juju consume admin/cos-model.prometheus-scrape prometheus
juju consume admin/cos-model.loki-logging loki
juju consume admin/cos-model.grafana-dashboards grafana

juju integrate grafana-agent prometheus
juju integrate grafana-agent loki
juju integrate grafana-agent grafana

# COS - Spark integration
juju switch spark-model
juju deploy prometheus-pushgateway-k8s --channel=edge; juju-wait
juju deploy prometheus-scrape-config-k8s scrape-interval-config --config scrape_interval=5; juj-wait
juju consume admin/cos-model.prometheus-scrape prometheus
juju integrate prometheus-pushgateway-k8s prometheus
juju integrate scrape-interval-config prometheus-pushgateway-k8s
juju integrate scrape-interval-config:metrics-endpoint prometheus:metrics-endpoint
PROMETHEUS_GATEWAY_IP=$(juju status --format=yaml | yq ".applications.prometheus-pushgateway-k8s.address")

# Download sample dataset from Kaggle
kaggle datasets download -d cityofLA/los-angeles-traffic-collision-data
unzip los-angeles-traffic-collision-data.zip
mc cp traffic-collision-data-from-2010-to-present.csv ceph-radosgw/data/

# configure Spark runtime - this UX will be improved soon by a charm
cat > spark.conf <<EOF
spark.eventLog.enabled=true
spark.hadoop.fs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider
spark.hadoop.fs.s3a.connection.ssl.enabled=true
spark.hadoop.fs.s3a.path.style.access=true
spark.hadoop.fs.s3a.access.key=${CEPH_ACCESS_KEY_ID}
spark.hadoop.fs.s3a.secret.key=${CEPH_SECRET_ACCESS_KEY}
spark.hadoop.fs.s3a.endpoint=https://${CEPH_IP}
spark.eventLog.dir=s3a://spark-history/spark-events/ 
spark.history.fs.logDirectory=s3a://spark-history/spark-events/
spark.driver.log.persistToDfs.enabled=true
spark.driver.log.dfsDir=s3a://spark-history/spark-events/
spark.metrics.conf.driver.sink.prometheus.pushgateway-address=${PROMETHEUS_GATEWAY_IP}:9091
spark.metrics.conf.driver.sink.prometheus.class=org.apache.spark.banzaicloud.metrics.sink.PrometheusSink
spark.metrics.conf.driver.sink.prometheus.enable-dropwizard-collector=true
spark.metrics.conf.driver.sink.prometheus.period=1
spark.metrics.conf.driver.sink.prometheus.metrics-name-capture-regex=([a-zA-Z0-9]*_[a-zA-Z0-9]*_[a-zA-Z0-9]*_)(.+)
spark.metrics.conf.driver.sink.prometheus.metrics-name-replacement=\$2
spark.metrics.conf.executor.sink.prometheus.pushgateway-address=${PROMETHEUS_GATEWAY_IP}:9091
spark.metrics.conf.executor.sink.prometheus.class=org.apache.spark.banzaicloud.metrics.sink.PrometheusSink
spark.metrics.conf.executor.sink.prometheus.enable-dropwizard-collector=true
spark.metrics.conf.executor.sink.prometheus.period=1
spark.metrics.conf.executor.sink.prometheus.metrics-name-capture-regex=([a-zA-Z0-9]*_[a-zA-Z0-9]*_[a-zA-Z0-9]*_)(.+)
spark.metrics.conf.executor.sink.prometheus.metrics-name-replacement=\$2
spark.kubernetes.executor.request.cores=0.01
spark.kubernetes.driver.request.cores=0.01
spark.kubernetes.container.image=ghcr.io/canonical/charmed-spark:3.4-22.04_edge
spark.executor.extraJavaOptions="-Djavax.net.ssl.trustStore=/spark-truststore/spark.truststore -Djavax.net.ssl.trustStorePassword=changeit"
spark.driver.extraJavaOptions="-Djavax.net.ssl.trustStore=/spark-truststore/spark.truststore -Djavax.net.ssl.trustStorePassword=changeit"
spark.kubernetes.executor.secrets.spark-truststore=/spark-truststore
spark.kubernetes.driver.secrets.spark-truststore=/spark-truststore
EOF

echo ${KUBECONF} > kubeconfig
cp /usr/lib/jvm/java-11-openjdk-amd64/lib/security/cacerts .
keytool -import -alias ceph-cert -file ceph-ca.pem -storetype JKS -keystore cacerts -storepass changeit -noprompt
mv cacerts spark.truststore
kubectl --kubeconfig=./kubeconfig --namespace=spark-model create secret generic spark-truststore --from-file spark.truststore

spark-client.service-account-registry create --username spark --namespace spark-model --primary --properties-file spark.conf --kubeconfig ./kubeconfig

spark-client.pyspark --username spark --namespace spark-model --conf spark.kubernetes.executor.request.cores=0.01 --conf spark.kubernetes.drive.request.cores=0.01 --conf spark.kubernetes.container.image=ghcr.io/canonical/charmed-spark:3.4-22.04_edge < pyspark-script.py

# Spawn various browser windows
# Spark History Server
juju switch spark-model
HISTORY_SERVER_URL=$(juju run traefik-k8s/leader show-proxied-endpoints | sed "s/proxied-endpoints: '//g" | sed "s/'//g" | jq -r '."spark-history-server-k8s".url')
google-chrome ${HISTORY_SERVER_URL}

# Grafana
juju switch cos-model
CMDOUT=$(juju run grafana/leader get-admin-password)
echo "admin/$(echo ${CMDOUT} | grep admin-password | awk -F: '{ print $2 }')"
GRAFANA_SERVER_URL=$(echo ${CMDOUT} | grep url | awk '{ print $2 }')

google-chrome ${GRAFANA_SERVER_URL}