add a trusted CA to all the worker nodes in a TKG cluster

README.md

Kubernetes - Private Registry

When you use your own custom private registry, you may end up using your own private CA signed or self signed certificates. This will cause container runtime to complain about untrusted cert, etc.

Here is a simple code that can fix this for you.

kubectl apply -f https://gist.github.com/yogendra/9937ead35a4ceb6f58ab5c4dc181cec3/node-ca-updater-daemons.yaml

node-ca-updater-daemonset.yaml

apiVersion: v1
data:
  ca.pem: |+
    -----BEGIN CERTIFICATE-----
    MIIElTCCA32gAwIBAgIJALXG1d19nJzcMA0GCSqGSIb3DQEBCwUAMIGMMQswCQYD
    VQQGEwJVUzELMAkGA1UECAwCQ0ExCzAJBgNVBAcMAkhQMREwDwYDVQQKDAhIYXJk
    dC5pbzEQMA4GA1UECwwHSG9tZWxhYjEZMBcGA1UEAwwQaG9tZWxhYi5oYXJkdC5p
    bzEjMCEGCSqGSIb3DQEJARYUaG9tZWxhYkByb2JoYXJkdC5jb20wHhcNMTkwNzA3
    MTkyMDE2WhcNMjIwNDI2MTkyMDE2WjCBjDELMAkGA1UEBhMCVVMxCzAJBgNVBAgM
    AkNBMQswCQYDVQQHDAJIUDERMA8GA1UECgwISGFyZHQuaW8xEDAOBgNVBAsMB0hv
    bWVsYWIxGTAXBgNVBAMMEGhvbWVsYWIuaGFyZHQuaW8xIzAhBgkqhkiG9w0BCQEW
    FGhvbWVsYWJAcm9iaGFyZHQuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
    CgKCAQEAzNAVcltNwy7wjxXOliRaWgBBG2ihZlw0peDc2XHJyUmgKUZx7ucGEKMn
    bK2HQ/qr7i59IY1l/tWocBoC5wolwzrzTuPg3oX4Rr491PIf0IDcZ5PrbRovf43m
    vVQsn/w+1TGzATr7eyaSOA9GhTkkT43FBEVXIhmlnupeXczaHOPsPMMD/5hVMYnT
    TvvoFNkT0lOr0QUKSHannyUpTCVU2+N5moKA/oCIWyZYwBgVcu1ieQ2Hj3HfdOVF
    jInbVGsWSZIHi2geeEIfYX0xZnt2U66/fSe9B0KPotDoQORHFMWDv8WMncplu+yy
    ukpTzJC68dl3tz5/9xfx9eoQJ35w+wIDAQABo4H3MIH0MA8GA1UdEwEB/wQFMAMB
    Af8wHQYDVR0OBBYEFNn4T5upQb9VfbtIYQG5h+w2L1OYMIHBBgNVHSMEgbkwgbaA
    FNn4T5upQb9VfbtIYQG5h+w2L1OYoYGSpIGPMIGMMQswCQYDVQQGEwJVUzELMAkG
    A1UECAwCQ0ExCzAJBgNVBAcMAkhQMREwDwYDVQQKDAhIYXJkdC5pbzEQMA4GA1UE
    CwwHSG9tZWxhYjEZMBcGA1UEAwwQaG9tZWxhYi5oYXJkdC5pbzEjMCEGCSqGSIb3
    DQEJARYUaG9tZWxhYkByb2JoYXJkdC5jb22CCQC1xtXdfZyc3DANBgkqhkiG9w0B
    AQsFAAOCAQEAAcfcrod95gsb05H3mJNm0esP4hE5FHoaaU4E1XdE3U9cAyyj4fEV
    J+DL6VGlbhB6nlhegTUpKjzqDei8dKfDlHDhNQLaUkGCYoVQIMmursALnOFxWraX
    i9Cl79jJAeALE+6vy0Gt2vNb12bPPFkaTWylTxLSih132rVPN6JCO42SOjzedGvr
    ydKMV/1JV9+bj3wb2N1k0rbvbrxbBs+fezWLiLcRbB1ZY/ATN7UlIzP0gutEghoT
    XN962HDhg//N/iFf+KZS0kYzvndv1Ze0Da1oWZBmmEj872bvVQi27f6lhNUA1Iuc
    RVDMhjpEYtrCRS0ecOkseoJ7knNrKI98Dg==
    -----END CERTIFICATE-----

kind: ConfigMap
metadata:
  name: trusted-ca-cm
  namespace: default


---
apiVersion: v1
data:
    build-ca.sh: |
      #!/usr/bin/env bash 
      set -euxo pipefail
      tdnf update 
      tdnf install -y ca-certificates
      tdnf install -y openssl-c_rehash
      echo \"$TRUSTED_CERT\" > /etc/ssl/certs/my-trusted-cert.pem
      /usr/bin/rehash_ca_certificates.sh
      curl -vv https://harbor.ingress.tkg.internal.hardt.io
      systemctl restart containerd
      
kind: ConfigMap
metadata:
    name: rehash-script
    namespace: default

---    

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: trusted-ca-updater
  namespace: default
  labels:
    k8s-app: trusted-ca-updater
spec:
  selector:
    matchLabels:
      name: trusted-ca-updater
  template:
    metadata:
      labels:
        name: trusted-ca-updater
    spec:
      tolerations:
      # this toleration is to have the daemonset runnable on master nodes
      # remove it if your masters can't run pods
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
      initContainers:
      - name: script-runner
        image: photon:3.0
        command: ["/bin/sh", "-c", "/root/build-ca.sh" ]
        volumeMounts:
        - name: update-trusted-certs-script
          mountPath: /root/
        - name: certs-dir
          mountPath: /etc/ssl/certs
        - name: agg-certs-dir
          mountPath: /etc/pki/tls/certs/
        env:
        - name: TRUSTED_CERT
          valueFrom:
            configMapKeyRef:
              name: trusted-ca-cm
              key: ca.pem    
        resources:
            limits:
              ephemeral-storage: 30G 
      containers:
      - name: sleepy
        image: photon:3.0
        command: ["/bin/sh"]
        args: ["-c", "while true; do sleep 3600;done"]
      volumes:
      - name: update-trusted-certs-script
        configMap:
            name: rehash-script
            defaultMode: 0766
      - name: certs-dir
        hostPath:
          path: /etc/ssl/certs
          type: Directory
      - name: agg-certs-dir
        hostPath:
          path: /etc/pki/tls/certs/
          type: Directory