client-secure.yaml

apiVersion: v1
kind: Pod
metadata:
  name: cockroachdb-client-secure
  namespace: crdb
  labels:
    app: cockroachdb-client
spec:
  serviceAccountName: cockroachdb
  initContainers:
  # The init-certs container sends a certificate signing request to the
  # kubernetes cluster.
  # You can see pending requests using: kubectl get csr
  # CSRs can be approved using:         kubectl certificate approve <csr name>
  #
  # In addition to the client certificate and key, the init-certs entrypoint will symlink
  # the cluster CA to the certs directory.
  - name: init-certs
    image: cockroachdb/cockroach-k8s-request-cert:0.4
    imagePullPolicy: IfNotPresent
    command:
    - "/bin/ash"
    - "-ecx"
    - "/request-cert -namespace=${POD_NAMESPACE} -certs-dir=/cockroach-certs -type=client -user=root -symlink-ca-from=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
    env:
    - name: POD_NAMESPACE
      valueFrom:
        fieldRef:
          fieldPath: metadata.namespace
    volumeMounts:
    - name: client-certs
      mountPath: /cockroach-certs
  containers:
  - name: cockroachdb-client
    image: cockroachdb/cockroach:v19.1.2
    imagePullPolicy: IfNotPresent
    volumeMounts:
    - name: client-certs
      mountPath: /cockroach-certs
    # Keep a pod open indefinitely so kubectl exec can be used to get a shell to it
    # and run cockroach client commands, such as cockroach sql, cockroach node status, etc.
    command:
    - sleep
    - "2147483648" # 2^31
  # This pod isn't doing anything important, so don't bother waiting to terminate it.
  terminationGracePeriodSeconds: 0
  volumes:
  - name: client-certs
    emptyDir: {}

cluster-init-secure.yaml

apiVersion: batch/v1
kind: Job
metadata:
  name: cluster-init-secure
  namespace: crdb
  labels:
    app: cockroachdb
spec:
  template:
    metadata:
      annotations:
        sidecar.istio.io/inject: "false"
    spec:
      serviceAccountName: cockroachdb
      initContainers:
      # The init-certs container sends a certificate signing request to the
      # kubernetes cluster.
      # You can see pending requests using: kubectl get csr
      # CSRs can be approved using:         kubectl certificate approve <csr name>
      #
      # In addition to the client certificate and key, the init-certs entrypoint will symlink
      # the cluster CA to the certs directory.
      - name: init-certs
        image: cockroachdb/cockroach-k8s-request-cert:0.4
        imagePullPolicy: IfNotPresent
        command:
        - "/bin/ash"
        - "-ecx"
        - "/request-cert -namespace=${POD_NAMESPACE} -certs-dir=/cockroach-certs -type=client -user=root -symlink-ca-from=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
        env:
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: client-certs
          mountPath: /cockroach-certs
      containers:
      - name: cluster-init
        image: cockroachdb/cockroach:v19.1.2
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - name: client-certs
          mountPath: /cockroach-certs
        command:
          - "/cockroach/cockroach"
          - "init"
          - "--certs-dir=/cockroach-certs"
          - "--host=cockroachdb-0.cockroachdb"
      restartPolicy: OnFailure
      volumes:
      - name: client-certs
        emptyDir: {}

cockroachdb-istio-README.txt

---
CREATE CLUSTER: 
gcloud container clusters create crdb-test \
  --enable-network-policy \
  --cluster-version latest \
  --num-nodes 4 \
  --machine-type=n1-standard-4 \
  --zone us-central1-a \
  --project skroovy

gcloud container clusters get-credentials crdb-test \
    --zone us-central1-a \
    --project skroovy

kubectl create clusterrolebinding cluster-admin-binding \
    --clusterrole=cluster-admin \
    --user=$(gcloud config get-value core/account)

---
INSTALL ISTIO:
curl -L https://git.io/getLatestIstio | ISTIO_VERSION=1.2.2 sh -

ISTIO CUSTOM RESOURCE DEFS:
kubectl create namespace istio-system

helm template $HOME/istio-1.2.2/install/kubernetes/helm/istio-init --name istio-init --namespace istio-system | kubectl apply -f -

CHECK CRDs:
kubectl get crds | grep 'istio.io\|certmanager.k8s.io' | wc -l
[23]

ISTIO 1.2.2 DEFAULT INSTALL:
helm template $HOME/istio-1.2.2/install/kubernetes/helm/istio --name istio --namespace istio-system | kubectl apply -f -

TEST:
kubectl get svc -n istio-system
kubectl get pods -n istio-system

---
CONFIGURE ISTIO:
BLOCK EGRESS BY DEFAULT:
kubectl get configmap istio -n istio-system -o yaml | grep -o "mode: ALLOW_ANY"
kubectl get configmap istio -n istio-system -o yaml | sed 's/mode: ALLOW_ANY/mode: REGISTRY_ONLY/g' | kubectl replace -n istio-system -f -

ALLOW POLICY CHECKS:
helm template $HOME/istio-1.2.2/install/kubernetes/helm/istio --namespace=istio-system -x templates/configmap.yaml \
    --set global.disablePolicyChecks=false | kubectl -n istio-system replace -f -

---
DEPLOY CRDB:
kubectl create namespace crdb
kubectl label namespace crdb istio-injection=enabled
kubectl create -f $HOME/Desktop/crdbTest/svcEntry-crdb.yaml
kubectl create -f $HOME/Desktop/crdbTest/cockroachdb-statefulset-secure.yaml

APPROVE CSR:
kubectl get csr -n crdb
[wait for crdb.node.cockroachdb-0 status: pending]

kubectl describe csr crdb.node.cockroachdb-0
kubectl certificate approve crdb.node.cockroachdb-0

[repeat for 1 and 2]

INIT CRDB CLUSTER:
kubectl get pods -n crdb
kubectl get persistentvolumes -n crdb

kubectl create -f $HOME/Desktop/crdbTest/cluster-init-secure.yaml
kubectl certificate approve crdb.client.root

kubectl get job cluster-init-secure -n crdb
kubectl get pods -n crdb

---
DEPLOY CRDB TEST CLIENT POD:
kubectl create -f $HOME/Desktop/crdbTest/client-secure.yaml

kubectl exec -it cockroachdb-client-secure -n crdb \
-- ./cockroach sql \
--certs-dir=/cockroach-certs \
--host=cockroachdb-public

CREATE DATABASE bank;
CREATE TABLE bank.accounts (id INT PRIMARY KEY, balance DECIMAL);
INSERT INTO bank.accounts VALUES (1, 1000.50);
SELECT * FROM bank.accounts;
CREATE USER roach WITH PASSWORD 'Q7gc8rEdS';
\q

---
ADMIN UI:
kubectl port-forward cockroachdb-0 8080 -n crdb
[Go to https://localhost:8080]

cockroachdb-statefulset-secure.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: cockroachdb
  namespace: crdb
  labels:
    app: cockroachdb
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
  name: cockroachdb
  namespace: crdb
  labels:
    app: cockroachdb
rules:
- apiGroups:
  - ""
  resources:
  - secrets
  verbs:
  - create
  - get
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: cockroachdb
  namespace: crdb
  labels:
    app: cockroachdb
rules:
- apiGroups:
  - certificates.k8s.io
  resources:
  - certificatesigningrequests
  verbs:
  - create
  - get
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
  name: cockroachdb
  namespace: crdb
  labels:
    app: cockroachdb
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: cockroachdb
subjects:
- kind: ServiceAccount
  name: cockroachdb
  namespace: crdb
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: cockroachdb
  namespace: crdb
  labels:
    app: cockroachdb
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cockroachdb
subjects:
- kind: ServiceAccount
  name: cockroachdb
  namespace: crdb
---
apiVersion: v1
kind: Service
metadata:
  # This service is meant to be used by clients of the database. It exposes a ClusterIP that will
  # automatically load balance connections to the different database pods.
  name: cockroachdb-public
  namespace: crdb
  labels:
    app: cockroachdb
spec:
  ports:
  # The main port, served by gRPC, serves Postgres-flavor SQL, internode
  # traffic and the cli.
  - port: 26257
    targetPort: 26257
    name: tcp-crdbpublic1
  # The secondary port serves the UI as well as health and debug endpoints.
  - port: 8080
    targetPort: 8080
    name: crdbpublic2
  selector:
    app: cockroachdb
---
apiVersion: v1
kind: Service
metadata:
  # This service only exists to create DNS entries for each pod in the stateful
  # set such that they can resolve each other's IP addresses. It does not
  # create a load-balanced ClusterIP and should not be used directly by clients
  # in most circumstances.
  name: cockroachdb
  namespace: crdb
  labels:
    app: cockroachdb
  annotations:
    # Use this annotation in addition to the actual publishNotReadyAddresses
    # field below because the annotation will stop being respected soon but the
    # field is broken in some versions of Kubernetes:
    # https://github.com/kubernetes/kubernetes/issues/58662
    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
    # Enable automatic monitoring of all instances when Prometheus is running in the cluster.
    prometheus.io/scrape: "true"
    prometheus.io/path: "_status/vars"
    prometheus.io/port: "8080"
spec:
  ports:
  - port: 26257
    targetPort: 26257
    name: tcp-crdbheadless1
  - port: 8080
    targetPort: 8080
    name: crdbheadless2
  # We want all pods in the StatefulSet to have their addresses published for
  # the sake of the other CockroachDB pods even before they're ready, since they
  # have to be able to talk to each other in order to become ready.
  publishNotReadyAddresses: true
  clusterIP: None
  selector:
    app: cockroachdb
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
  name: cockroachdb-budget
  namespace: crdb
  labels:
    app: cockroachdb
spec:
  selector:
    matchLabels:
      app: cockroachdb
  maxUnavailable: 1
---
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
  name: cockroachdb
  namespace: crdb
spec:
  serviceName: "cockroachdb"
  replicas: 3
  template:
    metadata:
      labels:
        app: cockroachdb
    spec:
      serviceAccountName: cockroachdb
      # Init containers are run only once in the lifetime of a pod, before
      # it's started up for the first time. It has to exit successfully
      # before the pod's main containers are allowed to start.
      initContainers:
      # The init-certs container sends a certificate signing request to the
      # kubernetes cluster.
      # You can see pending requests using: kubectl get csr
      # CSRs can be approved using:         kubectl certificate approve <csr name>
      #
      # All addresses used to contact a node must be specified in the --addresses arg.
      #
      # In addition to the node certificate and key, the init-certs entrypoint will symlink
      # the cluster CA to the certs directory.
      - name: init-certs
        image: cockroachdb/cockroach-k8s-request-cert:0.4
        imagePullPolicy: IfNotPresent
        command:
        - "/bin/ash"
        - "-ecx"
        - "/request-cert -namespace=${POD_NAMESPACE} -certs-dir=/cockroach-certs -type=node -addresses=localhost,127.0.0.1,$(hostname -f),$(hostname -f|cut -f 1-2 -d '.'),cockroachdb-public,cockroachdb-public.$(hostname -f|cut -f 3- -d '.') -symlink-ca-from=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
        env:
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: certs
          mountPath: /cockroach-certs
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - cockroachdb
              topologyKey: kubernetes.io/hostname
      containers:
      - name: cockroachdb
        image: cockroachdb/cockroach:v19.1.2
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 26257
#          name: grpc-crdb
        - containerPort: 8080
          name: crdb
        livenessProbe:
          httpGet:
            path: "/health"
            port: crdb
            scheme: HTTPS
          initialDelaySeconds: 30
          periodSeconds: 5
        readinessProbe:
          httpGet:
            path: "/health?ready=1"
            port: crdb
            scheme: HTTPS
          initialDelaySeconds: 10
          periodSeconds: 5
          failureThreshold: 2
        volumeMounts:
        - name: datadir
          mountPath: /cockroach/cockroach-data
        - name: certs
          mountPath: /cockroach/cockroach-certs
        env:
        - name: COCKROACH_CHANNEL
          value: kubernetes-secure
        command:
          - "/bin/bash"
          - "-ecx"
          # The use of qualified `hostname -f` is crucial:
          # Other nodes aren't able to look up the unqualified hostname.
          - "exec /cockroach/cockroach start --logtostderr --certs-dir /cockroach/cockroach-certs --advertise-host $(hostname -f) --http-addr 0.0.0.0 --join cockroachdb-0.cockroachdb,cockroachdb-1.cockroachdb,cockroachdb-2.cockroachdb --cache 25% --max-sql-memory 25%"
      # No pre-stop hook is required, a SIGTERM plus some time is all that's
      # needed for graceful shutdown of a node.
      terminationGracePeriodSeconds: 60
      volumes:
      - name: datadir
        persistentVolumeClaim:
          claimName: datadir
      - name: certs
        emptyDir: {}
  podManagementPolicy: Parallel
  updateStrategy:
    type: RollingUpdate
  volumeClaimTemplates:
  - metadata:
      name: datadir
    spec:
      accessModes:
        - "ReadWriteOnce"
      resources:
        requests:
          storage: 100Gi

svcEntry-crdb.yaml

apiVersion: networking.istio.io/v1alpha3
kind: ServiceEntry
metadata:
  name: crdb-stateful-service-entry
  namespace: crdb
spec:
  hosts:
  - "*.cockroachdb.crdb.svc.cluster.local"
  - "*.cockroachdb"
  location: MESH_INTERNAL
  ports:
  - number: 26257
    name: crdbheadless1
    protocol: TCP
  - number: 8080
    name: crdbheadless2
    protocol: HTTP
  resolution: NONE