Configure Upstream Vault and Install on k8s

configure-vault.md

prerequirements

• https://helm.sh/docs/intro/install/
• https://stedolan.github.io/jq/

install vault and spire on kubernetes

$ helm repo add hashicorp https://helm.releases.hashicorp.com
$ helm install vault hashicorp/vault

$ kubectl get pods
NAME                                    READY   STATUS    RESTARTS   AGE
vault-0                                 0/1     Running   0          18s
vault-agent-injector-7dd448d6c4-j6tn8   1/1     Running   0          18s

unseal vault

$ kubectl exec vault-0 -- vault operator init -key-shares=1 -key-threshold=1 -format=json > cluster-keys.json
$ VAULT_UNSEAL_KEY=$(cat cluster-keys.json | jq -r ".unseal_keys_b64[]")
$ kubectl exec vault-0 -- vault operator unseal $VAULT_UNSEAL_KEY

pod becomes ready

$ kubectl get pods
NAMESPACE            NAME                                               READY   STATUS    RESTARTS   AGE
default              vault-0                                            1/1     Running   0          69s
default              vault-agent-injector-7dd448d6c4-j6tn8              1/1     Running   0          69s

login

$  VAULT_TOKEN=$(cat cluster-keys.json | jq -r ".root_token")
$ kubectl exec vault-0 -- vault login $VAULT_TOKEN

configuire vault pki secret engine

$ kubectl exec vault-0 -- vault secrets enable pki
$ kubectl exec vault-0 -- vault secrets tune -max-lease-ttl=8760h pki

// you can adjust the settings to your preference
$ kubectl exec vault-0 -- vault write pki/root/generate/internal common_name=upsteream-ca.example.org ttl=8760h

create policy

$ cat <<EOF > spire.hcl
path "pki/root/sign-intermediate" {
  capabilities = ["update"]
}
EOF

$ kubectl cp spire.hcl vault-0:/tmp/.
$ kubectl exec vault-0 -- vault policy write spire /tmp/spire.hcl

configure approle auth method

$ kubectl exec vault-0 -- vault auth enable approle
// you can adjust the settings to your preference
$ kubectl exec vault-0 -- vault write auth/approle/role/spire  secret_id_ttl=120m  token_ttl=60m  token_max_tll=120m  policies="spire"

store approle credential as k8s secret

$ APPROLE_ID=$(kubectl exec vault-0 -- vault read --format json auth/approle/role/spire/role-id | jq -r .data.role_id)
$ SECRET_ID=$(kubectl exec vault-0 -- vault write --format json -f auth/approle/role/spire/secret-id | jq -r .data.secret_id)

$ kubectl create ns spire
$ kubectl create secret -n spire generic vault-credential --from-literal=approle_id=$APPROLE_ID --from-literal=secret_id=$SECRET_ID

deploy spire server

$  git clone [email protected]:spiffe/spire-examples.git
$ cd spire-examples

$ cat <<EOF > server-manifest.patch
index f292ec9..439fada 100644
--- a/examples/k8s/simple_psat/spire-server.yaml
+++ b/examples/k8s/simple_psat/spire-server.yaml
@@ -104,10 +104,11 @@ data:
         }
       }

-      UpstreamAuthority "disk" {
+      UpstreamAuthority "vault" {
         plugin_data {
-          key_file_path = "/run/spire/secrets/bootstrap.key"
-          cert_file_path = "/run/spire/config/bootstrap.crt"
+          vault_addr="http://vault.default.svc:8200/"
+          approle_auth {
+          }
         }
       }
     }
@@ -151,6 +152,17 @@ spec:
         - name: spire-server
           image: gcr.io/spiffe-io/spire-server:0.11.0
           args: ["-config", "/run/spire/config/server.conf"]
+          env:
+          - name: VAULT_APPROLE_ID
+            valueFrom:
+              secretKeyRef:
+                name: vault-credential
+                key: approle_id
+          - name: VAULT_APPROLE_SECRET_ID
+            valueFrom:
+              secretKeyRef:
+                name: vault-credential
+                key: secret_id
           ports:
             - containerPort: 8081
           volumeMounts:
EOF

$ patch -p1 < server-manifest.patch
$ kubectl apply -f examples/k8s/simple_psat/spire-server.yaml

// check spire certificate
$ kubectl port-forward -n spire spire-server-0 8081:8081 &
$ openssl s_client -connect localhost:8081 -showcerts

Using Token Auth

prerequirements

• https://helm.sh/docs/intro/install/
• https://stedolan.github.io/jq/

install vault and spire on kubernetes

$ helm repo add hashicorp https://helm.releases.hashicorp.com
$ helm install vault hashicorp/vault

$ kubectl get pods
NAME                                    READY   STATUS    RESTARTS   AGE
vault-0                                 0/1     Running   0          18s
vault-agent-injector-7dd448d6c4-j6tn8   1/1     Running   0          18s

unseal vault

$ kubectl exec vault-0 -- vault operator init -key-shares=1 -key-threshold=1 -format=json > cluster-keys.json
$ VAULT_UNSEAL_KEY=$(cat cluster-keys.jsaon | jq -r ".unseal_keys_b64[]")
$ kubectl exec vault-0 -- vault operator unseal $VAULT_UNSEAL_KEY

pod becomes ready

$ kubectl get pods
NAMESPACE            NAME                                               READY   STATUS    RESTARTS   AGE
default              vault-0                                            1/1     Running   0          69s
default              vault-agent-injector-7dd448d6c4-j6tn8              1/1     Running   0          69s

login

$  VAULT_TOKEN=$(cat cluster-keys.json | jq -r ".root_token")
$ kubectl exec vault-0 -- vault login $VAULT_TOKEN

configuire vault pki secret engine

$ kubectl exec vault-0 -- vault secrets enable pki
$ kubectl exec vault-0 -- vault secrets tune -max-lease-ttl=8760h pki

// you can adjust the settings to your preference
$ kubectl exec vault-0 -- vault write pki/root/generate/internal common_name=upsteream-ca.example.org ttl=8760h
$ kubectl exec vault-0 -- vault write pki/roles/spire allowed_domains=example.org allow_subdomains=true max_ttl=72h

create policy

$ cat <<EOF > spire.hcl
path "pki/root/sign-intermediate" {
  capabilities = ["update"]
}
EOF

$ kubectl cp spire.hcl vault-0:/tmp/.
$ kubectl exec vault-0 -- vault policy write spire /tmp/spire.hcl

generate token

$ TOKEN=$(kubectl exec vault-0 -- vault token create -policy=spire -format=json | jq -r .auth.client_token)

$ kubectl create ns spire
$ kubectl create secret -n spire generic vault-credential --from-literal=token=$TOKEN 

deploy spire server

$ git clone [email protected]:spiffe/spire-examples.git
$ cd spire-examples

$ cat <<EOF > server-manifest.patch
index f292ec9..8247f0f 100644
--- a/examples/k8s/simple_psat/spire-server.yaml
+++ b/examples/k8s/simple_psat/spire-server.yaml
@@ -104,10 +104,11 @@ data:
         }
       }

-      UpstreamAuthority "disk" {
+      UpstreamAuthority "vault" {
         plugin_data {
-          key_file_path = "/run/spire/secrets/bootstrap.key"
-          cert_file_path = "/run/spire/config/bootstrap.crt"
+          vault_addr="http://vault.default.svc:8200/"
+          token_auth {
+          }
         }
       }
     }
@@ -151,6 +152,12 @@ spec:
         - name: spire-server
           image: gcr.io/spiffe-io/spire-server:0.11.0
           args: ["-config", "/run/spire/config/server.conf"]
+          env:
+          - name: VAULT_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: vault-credential
+                key: token
           ports:
             - containerPort: 8081
           volumeMounts:
EOF

$ patch -p1 < server-manifest.patch
$ kubectl apply -f examples/k8s/simple_psat/spire-server.yaml

// check spire certificate
$ kubectl port-forward -n spire spire-server-0 8081:8081 &
$ openssl s_client -connect localhost:8081 -showcerts

Kuberntes Auth

kubectl exec vault-0 -- vault auth enable -path kubernetes-test kubernetes

// copy your k8s api-server's ca bundle.
kubectl cp ca.pem vault-0:/tmp/ca.pem

// you should specify issuer value according to your kube-apiserver configuration
kubectl exec vault-0 -- vault write auth/kubernetes-test/config issuer="https://kubernetes" token_reviewer_jwt="<Vault's SA TOKEN>" kubernetes_host="https://kubernetes" kubernetes_ca_cert=@/tmp/ca.pem
kubectl exec vault-0 -- vault write auth/kubernetes-test/role/my-role bound_service_account_names=spire-server bound_service_account_namespaces=spire policies=spire