Linode Kubernetes Add-ons Customized for RKE

linode-addons-secret.yaml

apiVersion: v1
kind: Secret
metadata:
  name: linode
  namespace: kube-system
stringData:
  token: ".... my secret token ...."
  region: ".... us-central, us-east, eu-west, etc..."

linode-addons.yml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: ccm-linode
  namespace: kube-system
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: system:ccm-linode
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  # TODO: make these permissions more fine-grained
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: ccm-linode
  namespace: kube-system
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: ccm-linode
  labels:
    app: ccm-linode
  namespace: kube-system
spec:
  selector:
    matchLabels:
      app: ccm-linode
  template:
    metadata:
      labels:
        app: ccm-linode
    spec:
      serviceAccountName: ccm-linode
      nodeSelector:
        # The CCM will only run on a Node labelled as a master, you may want to change this
        node-role.kubernetes.io/controlplane: "true"
      tolerations:
        # The CCM can run on Nodes tainted as masters
        - key: "node-role.kubernetes.io/controlplane"
          effect: "NoSchedule"
        # The CCM is a "critical addon"
        - key: "CriticalAddonsOnly"
          operator: "Exists"
        # This taint is set on all Nodes when an external CCM is used
        - key: node.cloudprovider.kubernetes.io/uninitialized
          value: "true"
          effect: NoSchedule
        - key: node.kubernetes.io/not-ready
          operator: Exists
          effect: NoSchedule
        - key: node.kubernetes.io/unreachable
          operator: Exists
          effect: NoSchedule
      hostNetwork: true
      containers:
        - image: linode/linode-cloud-controller-manager:latest
          imagePullPolicy: Always
          name: ccm-linode
          args:
          - --cloud-provider=linode
          - --v=3
          volumeMounts:
          - mountPath: /etc/kubernetes
            name: k8s
          env:
            - name: LINODE_API_TOKEN
              valueFrom:
                secretKeyRef:
                  name: linode
                  key: token
            - name: LINODE_REGION
              valueFrom:
                secretKeyRef:
                  name: linode
                  key: region
      volumes:
      - name: k8s
        hostPath:
          path: /etc/kubernetes
---
# pkg/linode-bs/deploy/kubernetes/01-csi-nodeinfo.yaml
# Requires CSINodeInfo feature gate (alpha in 1.12)
# xref: https://raw.githubusercontent.com/kubernetes/csi-api/master/pkg/crd/manifests/csidriver.yaml
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: csinodeinfos.csi.storage.k8s.io
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
spec:
  group: csi.storage.k8s.io
  version: v1alpha1
  names:
    kind: CSINodeInfo
    plural: csinodeinfos
  scope: Cluster
  validation:
    openAPIV3Schema:
      properties:
        spec:
          description: Specification of CSINodeInfo
          properties:
            drivers:
              description: List of CSI drivers running on the node and their specs.
              type: array
              items:
                properties:
                  name:
                    description: The CSI driver that this object refers to.
                    type: string
                  nodeID:
                    description: The node from the driver point of view.
                    type: string
                  topologyKeys:
                    description: List of keys supported by the driver.
                    items:
                      type: string
                    type: array
        status:
          description: Status of CSINodeInfo
          properties:
            drivers:
              description: List of CSI drivers running on the node and their statuses.
              type: array
              items:
                properties:
                  name:
                    description: The CSI driver that this object refers to.
                    type: string
                  available:
                    description: Whether the CSI driver is installed.
                    type: boolean
                  volumePluginMechanism:
                    description: Indicates to external components the required mechanism
                      to use for any in-tree plugins replaced by this driver.
                    pattern: in-tree|csi
                    type: string

---
# pkg/linode-bs/deploy/kubernetes/02-csi-driver.yaml
# Requires CSIDriverRegistry feature gate (alpha in 1.12)
# xref: https://raw.githubusercontent.com/kubernetes/csi-api/master/pkg/crd/manifests/csinodeinfo.yaml
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: csidrivers.csi.storage.k8s.io
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
spec:
  version: v1alpha1
  group: csi.storage.k8s.io
  names:
    kind: CSIDriver
    plural: csidrivers
  scope: Cluster
  validation:
    openAPIV3Schema:
      properties:
        spec:
          description: Specification of the CSI Driver.
          properties:
            attachRequired:
              description: Indicates this CSI volume driver requires an attach operation,
                and that Kubernetes should call attach and wait for any attach operation
                to complete before proceeding to mount.
              type: boolean
            podInfoOnMountVersion:
              description: Indicates this CSI volume driver requires additional pod
                information (like podName, podUID, etc.) during mount operations.
              type: string
---
# pkg/linode-bs/deploy/kubernetes/03-accounts-roles-bindings.yaml
##### Node Service Account, Roles, RoleBindings
apiVersion: v1
kind: ServiceAccount
metadata:
  name: csi-node-sa
  namespace: kube-system

---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: driver-registrar-role
  namespace: kube-system
rules:
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]


---

kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: driver-registrar-binding
  namespace: kube-system
subjects:
  - kind: ServiceAccount
    name: csi-node-sa
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: driver-registrar-role
  apiGroup: rbac.authorization.k8s.io

---
##### Controller Service Account, Roles, Rolebindings
apiVersion: v1
kind: ServiceAccount
metadata:
  name: csi-controller-sa
  namespace: kube-system

---
# xref: https://github.com/kubernetes-csi/external-provisioner/blob/master/deploy/kubernetes/rbac.yaml
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: external-provisioner-role
  namespace: kube-system
rules:
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get", "list"]
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["list", "watch", "create", "update", "patch"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshots"]
    verbs: ["get", "list"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshotcontents"]
    verbs: ["get", "list"]

---

kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: csi-controller-provisioner-binding
  namespace: kube-system
subjects:
  - kind: ServiceAccount
    name: csi-controller-sa
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: external-provisioner-role
  apiGroup: rbac.authorization.k8s.io

---
# xref: https://github.com/kubernetes-csi/external-attacher/blob/master/deploy/kubernetes/rbac.yaml
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: external-attacher-role
  namespace: kube-system
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: [""]
    resources: ["nodes"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["csi.storage.k8s.io"]
    resources: ["csinodeinfos"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["volumeattachments"]
    verbs: ["get", "list", "watch", "update"]

---

kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: csi-controller-attacher-binding
  namespace: kube-system
subjects:
  - kind: ServiceAccount
    name: csi-controller-sa
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: external-attacher-role
  apiGroup: rbac.authorization.k8s.io

---
# xref: https://github.com/kubernetes-csi/external-snapshotter/blob/master/deploy/kubernetes/rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: external-snapshotter-role
  namespace: kube-system
rules:
- apiGroups: ["snapshot.storage.k8s.io"]
  resources: ["volumesnapshotclasses"]
  verbs: ["get", "list", "watch"]
- apiGroups: ["snapshot.storage.k8s.io"]
  resources: ["volumesnapshotcontents"]
  verbs: ["create", "get", "list", "watch", "update", "delete"]
- apiGroups: ["snapshot.storage.k8s.io"]
  resources: ["volumesnapshots"]
  verbs: ["get", "list", "watch", "update"]
- apiGroups: ["apiextensions.k8s.io"]
  resources: ["customresourcedefinitions"]
  verbs: ["create", "list", "watch", "delete"]
- apiGroups: [""]
  resources: ["events"]
  verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: ["storage.k8s.io"]
  resources: ["storageclasses"]
  verbs: ["watch", "get", "list"]
- apiGroups: ["admissionregistration.k8s.io"]
  resources: ["mutatingwebhookconfigurations"]
  verbs: ["create"]

---

kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: csi-controller-snapshotter-binding
  namespace: kube-system
subjects:
  - kind: ServiceAccount
    name: csi-controller-sa
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: external-snapshotter-role
  apiGroup: rbac.authorization.k8s.io

---
# pkg/linode-bs/deploy/kubernetes/04-csi-driver-instance.yaml

apiVersion: csi.storage.k8s.io/v1alpha1
kind: CSIDriver
metadata:
  name: linodebs.csi.linode.com
spec:
  attachRequired: true
  podInfoOnMountVersion: "v1"

---
# pkg/linode-bs/deploy/kubernetes/05-csi-storageclass.yaml
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: linode-block-storage
  namespace: kube-system
  annotations:
    storageclass.kubernetes.io/is-default-class: "true"
provisioner: linodebs.csi.linode.com
---
# pkg/linode-bs/deploy/kubernetes/06-ss-csi-linode-controller.yaml
# stateful sets needs a headless service to provide pod network indentity
kind: StatefulSet
apiVersion: apps/v1
metadata:
  name: csi-linode-controller
  namespace: kube-system
spec:
  serviceName: "csi-linode"
  replicas: 1
  selector:
    matchLabels:
      app: csi-linode-controller
  template:
    metadata:
      labels:
        app: csi-linode-controller
        role: csi-linode
    spec:
      serviceAccount: csi-controller-sa
      containers:
        - name: csi-provisioner
          image: quay.io/k8scsi/csi-provisioner:v1.0.0
          args:
            - "--provisioner=linodebs.csi.linode.com"
            - "--csi-address=$(ADDRESS)"
            - "--v=5"
          env:
            - name: ADDRESS
              value: /var/lib/csi/sockets/pluginproxy/csi.sock
          imagePullPolicy: "Always"
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy/
        - name: csi-attacher
          image: quay.io/k8scsi/csi-attacher:v1.0.0
          args:
            - "--v=5"
            - "--csi-address=$(ADDRESS)"
          env:
            - name: ADDRESS
              value: /var/lib/csi/sockets/pluginproxy/csi.sock
          imagePullPolicy: "Always"
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy/
        - name: linode-csi-plugin
          image: linode/linode-blockstorage-csi-driver:canary
          args :
            - "--endpoint=$(CSI_ENDPOINT)"
            - "--token=$(LINODE_TOKEN)"
            - "--url=$(LINODE_API_URL)"
            - "--node=$(NODE_NAME)"
            - "--v=10"
          env:
            - name: CSI_ENDPOINT
              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
            - name: LINODE_API_URL
              value: https://api.linode.com/v4
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: LINODE_TOKEN
              valueFrom:
                secretKeyRef:
                  name: linode
                  key: token
          imagePullPolicy: "Always"
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy/
      volumes:
        - name: socket-dir
          emptyDir: {}
---
# pkg/linode-bs/deploy/kubernetes/07-ds-csi-linode-node.yaml
kind: DaemonSet
apiVersion: extensions/v1beta1
metadata:
  name: csi-linode-node
  namespace: kube-system
spec:
  selector:
    matchLabels:
      app: csi-linode-node
  template:
    metadata:
      labels:
        app: csi-linode-node
        role: csi-linode
    spec:
      serviceAccount: csi-node-sa
      hostNetwork: true
      containers:
        - name: driver-registrar
          image: quay.io/k8scsi/driver-registrar:v1.0-canary
          args:
            - "--v=5"
            - "--csi-address=$(ADDRESS)"
            - "--mode=node-register"
            - "--driver-requires-attachment=true"
            - "--pod-info-mount-version=\"v1\""
            - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
          env:
            - name: ADDRESS
              value: /csi/csi.sock
            - name: DRIVER_REG_SOCK_PATH
              value: /var/lib/kubelet/plugins/linodebs.csi.linode.com/csi.sock
            - name: KUBE_NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          volumeMounts:
            - name: plugin-dir
              mountPath: /csi/
            - name: registration-dir
              mountPath: /registration/
        - name: csi-linode-plugin
          image: linode/linode-blockstorage-csi-driver:canary
          args :
            - "--endpoint=$(CSI_ENDPOINT)"
            - "--token=$(LINODE_TOKEN)"
            - "--url=$(LINODE_API_URL)"
            - "--node=$(NODE_NAME)"
            - "--v=10"
          env:
            - name: CSI_ENDPOINT
              value: unix:///csi/csi.sock
            - name: LINODE_API_URL
              value: https://api.linode.com/v4
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: LINODE_TOKEN
              valueFrom:
                secretKeyRef:
                  name: linode
                  key: token
          imagePullPolicy: "Always"
          securityContext:
            privileged: true
            capabilities:
              add: ["SYS_ADMIN"]
            allowPrivilegeEscalation: true
          volumeMounts:
            - name: plugin-dir
              mountPath: /csi
            - name: pods-mount-dir
              mountPath: /var/lib/kubelet
              # needed so that any mounts setup inside this container are
              # propagated back to the host machine.
              mountPropagation: "Bidirectional"
            - mountPath: /dev
              name: device-dir
      volumes:
        - name: registration-dir
          hostPath:
            path: /var/lib/kubelet/plugins_registry/
            type: DirectoryOrCreate
        - name: kubelet-dir
          hostPath:
            path: /var/lib/kubelet
            type: Directory
        - name: plugin-dir
          hostPath:
            path: /var/lib/kubelet/plugins/linodebs.csi.linode.com
            type: DirectoryOrCreate
        - name: pods-mount-dir
          hostPath:
            path: /var/lib/kubelet
            type: Directory
        - name: device-dir
          hostPath:
            path: /dev
        # The following mounts are required to trigger host udevadm from container
        - name: udev-rules-etc
          hostPath:
            path: /etc/udev
            type: Directory
        - name: udev-rules-lib
          hostPath:
            path: /lib/udev
            type: Directory
        - name: udev-socket
          hostPath:
            path: /run/udev
            type: Directory
        - name: sys
          hostPath:
            path: /sys
            type: Directory
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: external-dns
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: external-dns
  namespace: kube-system
rules:
- apiGroups: [""]
  resources: ["services"]
  verbs: ["get","watch","list"]
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["get","watch","list"]
- apiGroups: ["extensions"]
  resources: ["ingresses"]
  verbs: ["get","watch","list"]
- apiGroups: [""]
  resources: ["nodes"]
  verbs: ["list"]
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: external-dns-viewer
  namespace: kube-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: external-dns
subjects:
- kind: ServiceAccount
  name: external-dns
  namespace: kube-system
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: external-dns
  namespace: kube-system
spec:
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: external-dns
    spec:
      serviceAccountName: external-dns
      containers:
      - name: external-dns
        image: registry.opensource.zalan.do/teapot/external-dns:latest
        args:
        - --provider=linode
        # - --domain-filter=example.com # (optional) limit to only example.com domains; change to match the zone created above.
        env:
        - name: EXTERNAL_DNS_SOURCE
          value: |-
            service
            ingress
        - name: LINODE_TOKEN
          valueFrom:
            secretKeyRef:
              name: linode
              key: token
---