kubernetesmastercustomdata.yml

#cloud-config

# { { { variable } } }

packages:
 - etcd
 - jq
 - traceroute

runcmd:
- /bin/bash -c "/bin/echo DAEMON_ARGS=--advertise-client-urls ""http://127.0.0.1:2379,http://{{{masterPrivateIp}}}:2379"" --listen-client-urls ""http://0.0.0.0:2379,http://0.0.0.0:4001"" | tee -a /etc/default/etcd"
- /usr/bin/curl -sSL --retry 12 --retry-delay 10 https://get.docker.com/ > /tmp/install-docker
- /bin/bash -c "/bin/bash /tmp/install-docker"
- /usr/bin/curl -sSL --retry 12 --retry-delay 10 https://storage.googleapis.com/kubernetes-release/release/{{{kubectlVersion}}}/bin/linux/amd64/kubectl > /usr/local/bin/kubectl
- chmod +x /usr/local/bin/kubectl

write_files:
- path: "/etc/systemd/system/docker.service.d/clear_mount_propagation_flags.conf"
  permissions: "0644"
  owner: "root"
  content: |
    [Service]
    MountFlags=shared

- path: "/etc/systemd/system/docker.service.d/overlay.conf"
  permissions: "0644"
  owner: "root"
  content: |
    [Service]
    ExecStart=
    ExecStart=/usr/bin/docker daemon -H fd:// --storage-driver=overlay

- path: "/etc/kubernetes/certs/ca.crt"
  permissions: "0644"
  encoding: "base64"
  owner: "root"
  content: |
    {{{caCertificate}}}

- path: "/etc/kubernetes/certs/apiserver.crt"
  permissions: "0644"
  encoding: "base64"
  owner: "root"
  content: |
    {{{apiServerCertificate}}}

- path: "/etc/kubernetes/certs/client.crt"
  permissions: "0644"
  encoding: "base64"
  owner: "root"
  content: |
    {{{clientCertificate}}}

- path: "/var/lib/kubelet/kubeconfig"
  permissions: "0644"
  owner: "root"
  content: |
    apiVersion: v1
    kind: Config
    clusters:
    - name: localcluster
      cluster:
        certificate-authority: /etc/kubernetes/certs/ca.crt
        server: https://{{{masterPrivateIp}}}:443
    users:
    - name: client
      user:
        client-certificate: /etc/kubernetes/certs/client.crt
        client-key: /etc/kubernetes/certs/client.key
    contexts:
    - context:
        cluster: localcluster
        user: client
      name: localclustercontext
    current-context: localclustercontext

- path: /etc/kubernetes/manifests/kube-apiserver.yaml
  permissions: "0644"
  owner: "root"
  content: |
      apiVersion: "v1"
      kind: "Pod"
      metadata:
        name: "kube-apiserver"
        namespace: "kube-system"
        labels:
          tier: control-plane
          component: kube-apiserver
      spec:
        hostNetwork: true
        containers:
          - name: "kube-apiserver"
            image: "{{{kubernetesHyperkubeSpec}}}"
            command: 
              - "/hyperkube"
              - "apiserver"
              - "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota"
              - "--address=0.0.0.0"
              - "--allow-privileged"
              - "--insecure-port=8080"
              - "--secure-port=443"
              - "--cloud-provider=azure"
              - "--cloud-config=/etc/kubernetes/azure.json"
              - "--service-cluster-ip-range={{{kubeServiceCidr}}}"
              - "--etcd-servers=http://127.0.0.1:4001"
              - "--tls-cert-file=/etc/kubernetes/certs/apiserver.crt"
              - "--tls-private-key-file=/etc/kubernetes/certs/apiserver.key"
              - "--client-ca-file=/etc/kubernetes/certs/ca.crt"
              - "--service-account-key-file=/etc/kubernetes/certs/apiserver.key"
              - "--v=4"
              - "--runtime-config=batch/v2alpha1"
            volumeMounts:
              - name: "etc-kubernetes"
                mountPath: "/etc/kubernetes"
              - name: "var-lib-kubelet"
                mountPath: "/var/lib/kubelet"
        volumes:
          - name: "etc-kubernetes"
            hostPath:
              path: "/etc/kubernetes"
          - name: "var-lib-kubelet"
            hostPath:
              path: "/var/lib/kubelet"

- path: /etc/kubernetes/manifests/kube-controller-manager.yaml
  permissions: "0644"
  owner: "root"
  content: |
      apiVersion: "v1"
      kind: "Pod"
      metadata:
        name: "kube-controller-manager"
        namespace: "kube-system"
        labels:
          tier: control-plane
          component: kube-controller-manager
      spec:
        hostNetwork: true
        containers:
          - name: "kube-controller-manager"
            image: "{{{kubernetesHyperkubeSpec}}}"
            command: 
              - "/hyperkube"
              - "controller-manager"
              - "--master=127.0.0.1:8080"
              - "--kubeconfig=/var/lib/kubelet/kubeconfig"
              - "--allocate-node-cidrs=true"
              - "--cluster-cidr=10.244.0.0/16"
              - "--cluster-name={{{masterFqdnPrefix}}}"
              - "--cloud-provider=azure"
              - "--cloud-config=/etc/kubernetes/azure.json"
              - "--root-ca-file=/etc/kubernetes/certs/ca.crt"
              - "--service-account-private-key-file=/etc/kubernetes/certs/apiserver.key"
              - "--v=2"
            volumeMounts: 
              - name: "etc-kubernetes"
                mountPath: "/etc/kubernetes"
              - name: "var-lib-kubelet"
                mountPath: "/var/lib/kubelet"
        volumes:
          - name: "etc-kubernetes"
            hostPath:
              path: "/etc/kubernetes"
          - name: "var-lib-kubelet"
            hostPath:
              path: "/var/lib/kubelet"

- path: /etc/kubernetes/manifests/kube-scheduler.yaml
  permissions: "0644"
  owner: "root"
  content: |
      apiVersion: "v1"
      kind: "Pod"
      metadata:
        name: "kube-scheduler"
        namespace: "kube-system"
        labels:
          tier: control-plane
          component: kube-scheduler
      spec:
        hostNetwork: true
        containers:
          - name: "kube-scheduler"
            image: "{{{kubernetesHyperkubeSpec}}}"
            command:
              - "/hyperkube"
              - "scheduler"
              - "--master=127.0.0.1:8080"
              - "--kubeconfig=/var/lib/kubelet/kubeconfig"
              - "--v=2"
            volumeMounts:
              - name: "etc-kubernetes"
                mountPath: "/etc/kubernetes"
              - name: "var-lib-kubelet"
                mountPath: "/var/lib/kubelet"
        volumes:
          - name: "etc-kubernetes"
            hostPath:
              path: "/etc/kubernetes"
          - name: "var-lib-kubelet"
            hostPath:
              path: "/var/lib/kubelet"

- path: /etc/kubernetes/manifests/kube-addon-manager.yml
  permissions: "0644"
  owner: "root"
  content: |
    apiVersion: v1
    kind: Pod
    metadata:
      name: kube-addon-manager
      namespace: kube-system
      version: v1
    spec:
      hostNetwork: true
      containers:
      - name: kube-addon-manager
        # TODO: consider if hardcoding this is the right thing to do...
        image: gcr.io/google_containers/kube-addon-manager-amd64:v6
        resources:
          requests:
            cpu: 5m
            memory: 50Mi
        volumeMounts:
        - name: addons
          mountPath: "/etc/kubernetes/addons"
          readOnly: true
      volumes:
      - name: addons
        hostPath:
          path: "/etc/kubernetes/addons"

- path: /etc/kubernetes/addons/kube-dns-service.yaml
  permissions: "0644"
  encoding: gzip
  owner: "root"
  content: !!binary |
    MASTER_ADDON_KUBE_DNS_SERVICE_B64_GZIP_STR

- path: /etc/kubernetes/addons/kube-dns-deployment.yaml
  permissions: "0644"
  encoding: gzip
  owner: "root"
  content: !!binary |
    MASTER_ADDON_KUBE_DNS_DEPLOYMENT_B64_GZIP_STR

- path: /etc/kubernetes/addons/kube-proxy-daemonset.yaml
  permissions: "0644"
  encoding: gzip
  owner: "root"
  content: !!binary |
    MASTER_ADDON_KUBE_PROXY_DAEMONSET_B64_GZIP_STR

- path: /etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml
  permissions: "0644"
  encoding: gzip
  owner: "root"
  content: !!binary |
    MASTER_ADDON_KUBERNETES_DASHBOARD_DEPLOYMENT_B64_GZIP_STR

- path: /etc/kubernetes/addons/kubernetes-dashboard-service.yaml
  permissions: "0644"
  encoding: gzip
  owner: "root"
  content: !!binary |
    MASTER_ADDON_KUBERNETES_DASHBOARD_SERVICE_B64_GZIP_STR

- path: /etc/kubernetes/addons/kube-heapster-service.yaml
  permissions: "0644"
  encoding: gzip
  owner: "root"
  content: !!binary |
    MASTER_ADDON_HEAPSTER_SERVICE_B64_GZIP_STR

- path: /etc/kubernetes/addons/kube-heapster-deployment.yaml
  permissions: "0644"
  encoding: gzip
  owner: "root"
  content: !!binary |
    MASTER_ADDON_HEAPSTER_DEPLOYMENT_B64_GZIP_STR

- path: "/etc/systemd/system/kubelet.service"
  permissions: "0644"
  owner: "root"
  content: |
    [Unit]
    Description=Kubelet
    Requires=docker.service
    After=docker.service

    [Service]
    Restart=always
    ExecStartPre=/bin/mkdir -p /var/lib/kubelet
    ExecStartPre=/bin/sed -i "s|<kubernetesHyperkubeSpec>|{{{kubernetesHyperkubeSpec}}}|g" "/etc/kubernetes/addons/kube-proxy-daemonset.yaml"
    ExecStartPre=/bin/mount --bind /var/lib/kubelet /var/lib/kubelet
    ExecStartPre=/bin/mount --make-shared /var/lib/kubelet
    ExecStart=/usr/bin/docker run \
      --net=host \
      --pid=host \
      --privileged \
      --volume=/dev:/dev \
      --volume=/sys:/sys:ro \
      --volume=/var/run:/var/run:rw \
      --volume=/var/lib/docker/:/var/lib/docker:rw \
      --volume=/var/lib/kubelet/:/var/lib/kubelet:shared \
      --volume=/var/log:/var/log:rw \
      --volume=/etc/kubernetes/:/etc/kubernetes:ro \
      --volume=/srv/kubernetes/:/srv/kubernetes:ro \
        {{{kubernetesHyperkubeSpec}}} \
          /hyperkube kubelet \
            --api-servers="https://{{{masterPrivateIp}}}:443" \
            --kubeconfig=/var/lib/kubelet/kubeconfig \
            --address=0.0.0.0 \
            --allow-privileged=true \
            --enable-server \
            --enable-debugging-handlers \
            --config=/etc/kubernetes/manifests \
            --cluster-dns={{{kubeDnsServiceIP}}} \
            --cluster-domain=cluster.local \
            --register-schedulable=false \
            --cloud-provider=azure \
            --cloud-config=/etc/kubernetes/azure.json \
            --v=2
    ExecStop=/usr/bin/docker stop -t 2 kubelet

    [Install]
    WantedBy=multi-user.target

- path: "/opt/azure/containers/provision.sh"
  permissions: "0744"
  encoding: gzip
  owner: "root"
  content: !!binary |
    MASTER_PROVISION_B64_GZIP_STR