Skip to content

Instantly share code, notes, and snippets.

@joejulian

joejulian/provider-components.yaml

Created September 14, 2018 00:05
Show Gist options
  • Save joejulian/9b912825d04f72a5e9aeaf94e7bf3099 to your computer and use it in GitHub Desktop.
Save joejulian/9b912825d04f72a5e9aeaf94e7bf3099 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
provider-components.yaml
---
apiVersion: apps/v1beta1
kind: Deployment
metadata:
name: clusterapi-controllers
labels:
api: clusterapi
spec:
replicas: 1
template:
metadata:
labels:
api: clusterapi
spec:
nodeSelector:
node-role.kubernetes.io/master: ""
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
- key: CriticalAddonsOnly
operator: Exists
- effect: NoExecute
key: node.alpha.kubernetes.io/notReady
operator: Exists
- effect: NoExecute
key: node.alpha.kubernetes.io/unreachable
operator: Exists
containers:
- name: controller-manager
image: gcr.io/k8s-cluster-api/controller-manager:0.0.8
volumeMounts:
- name: config
mountPath: /etc/kubernetes
- name: certs
mountPath: /etc/ssl/certs
command:
- "./controller-manager"
args:
- --kubeconfig=/etc/kubernetes/admin.conf
- --leader-elect
resources:
requests:
cpu: 100m
memory: 20Mi
limits:
cpu: 100m
memory: 30Mi
- name: ssh-cluster-controller
image: quay.io/samsung_cnct/ssh-cluster-controller:prod
imagePullPolicy: Always
volumeMounts:
- name: config
mountPath: /etc/kubernetes
- name: certs
mountPath: /etc/ssl/certs
- name: machine-setup
mountPath: /etc/machinesetup
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
command:
- "./cluster-controller"
args:
- --kubeconfig=/etc/kubernetes/admin.conf
- --leader-elect
resources:
requests:
cpu: 200m
memory: 200Mi
limits:
cpu: 400m
memory: 500Mi
- name: ssh-machine-controller
image: quay.io/samsung_cnct/ssh-machine-controller:prod
imagePullPolicy: Always
volumeMounts:
- name: config
mountPath: /etc/kubernetes
- name: certs
mountPath: /etc/ssl/certs
- name: machine-setup
mountPath: /etc/machinesetup
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
command:
- "./machine-controller"
args:
- --kubeconfig=/etc/kubernetes/admin.conf
- --machinesetup=/etc/machinesetup/machine_setup_configs.yaml
- --leader-elect
resources:
requests:
cpu: 200m
memory: 200Mi
limits:
cpu: 400m
memory: 500Mi
volumes:
- name: config
hostPath:
path: /etc/kubernetes
- name: certs
hostPath:
path: /etc/ssl/certs
- name: machine-setup
configMap:
name: machine-setup
---
apiVersion: v1
kind: Secret
type: Opaque
metadata:
name: cluster-private-key
namespace: default
data:
private-key: LS0tLS1CRUdJTiBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0KYjNCbGJuTnphQzFyWlhrdGRqRUFBQUFBQkc1dmJtVUFBQUFFYm05dVpRQUFBQUFBQUFBQkFBQUFNd0FBQUF0emMyZ3RaVwpReU5UVXhPUUFBQUNESHJxUzgweFpud2pwaE9QaVB1WXZsSm5LRWE5YzkwbkUvNnhyNnhxeU9Ed0FBQUppSE1CdjNoekFiCjl3QUFBQXR6YzJndFpXUXlOVFV4T1FBQUFDREhycVM4MHhabndqcGhPUGlQdVl2bEpuS0VhOWM5MG5FLzZ4cjZ4cXlPRHcKQUFBRURUK1VGNHhsdlRDeEhjY29IYmt2SEhURUNwMFVyNHNaZjIxVFVhREUwZnlNZXVwTHpURm1mQ09tRTQrSSs1aStVbQpjb1JyMXozU2NUL3JHdnJHckk0UEFBQUFFVU5PUTFRZ1RVRkJVeUJxYWkxamJtTjBBUUlEQkE9PQotLS0tLUVORCBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0K
pass-phrase: ""
---
apiVersion: v1
kind: ConfigMap
metadata:
name: machine-setup
data:
machine_setup_configs.yaml: |-
items:
- machineParams:
roles:
- Master
- Etcd
versions:
kubelet: 1.10.6
controlPlane: 1.10.6
metadata:
startupScript: |
set -e
set -x
(
ARCH=amd64
function add_kubernetes_repo () {
sudo cp /dev/stdin /etc/yum.repos.d/kubernetes.repo <<< "
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
"
}
function prips () {
cidr=$1
# range is bounded by network (-n) & broadcast (-b) addresses.
lo=$(ipcalc -n $cidr |cut -f2 -d=)
hi=$(ipcalc -b $cidr |cut -f2 -d=)
read a b c d <<< $(echo $lo |tr . ' ')
read e f g h <<< $(echo $hi |tr . ' ')
eval "echo {$a..$e}.{$b..$f}.{$c..$g}.{$d..$h}"
}
# kubeadm uses 10th IP as DNS server
CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | awk '{print $11}')
KUBELET="kubelet-${KUBELET_VERSION}"
KUBEADM="kubeadm-${KUBELET_VERSION}"
add_kubernetes_repo
sudo yum install -y \
docker \
${KUBELET} \
${KUBEADM}
sudo cp /dev/stdin /etc/sysconfig/docker <<< 'DOCKER_OPTS="--iptables=false --ip-masq=false"'
sudo cp /dev/stdin /etc/systemd/system/kubelet.service.d/20-kubenet.conf <<< '[Service]
Environment="KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-domain=${CLUSTER_DNS_DOMAIN}"
'
# Not yet selinux ready
sudo setenforce 0
sudo systemctl daemon-reload
sudo systemctl disable --now firewalld
sudo systemctl enable --now docker kubelet
# Set up kubeadm config file to pass parameters to kubeadm init.
# Note the latest API version for kubeadm is v1alphav2 as of kubeadm
# 1.11. We need to account for these different versions in the next
# rewrite of this script.
sudo cp /dev/stdin /etc/kubernetes/kubeadm_config.yaml <<< "
apiVersion: kubeadm.k8s.io/v1alpha1
kind: MasterConfiguration
api:
advertiseAddress: ${MASTER_IP}
bindPort: 443
etcd:
local:
dataDir: /var/lib/etcd
image:
kubernetesVersion: v${CONTROL_PLANE_VERSION}
token: ${TOKEN}
kubeProxy:
config:
clusterCIDR: ${POD_CIDR}
networking:
dsnDomain: ${CLUSTER_DNS_DOMAIN}
serviceSubnet: ${SERVICE_CIDR}
podSubnet: ${POD_CIDR}
"
# Create and set bridge-nf-call-iptables to 1 to pass the kubeadm preflight check.
# Workaround was found here:
# http://zeeshanali.com/sysadmin/fixed-sysctl-cannot-stat-procsysnetbridgebridge-nf-call-iptables/
sudo modprobe br_netfilter
# [ERROR Swap]: running with swap on is not supported. Please disable swap
# this was put in place since its something that must be done on ubuntu machines
# when provisioning for use with kubeadm. Note, kubelet requires this to be off
# and may be something we remove in the future and leave it as part of provisioning
sudo swapoff -a
sudo kubeadm init --config /etc/kubernetes/kubeadm_config.yaml
for tries in $(seq 1 60); do
sudo kubectl --kubeconfig /etc/kubernetes/kubelet.conf annotate --overwrite node $(hostname) machine=${MACHINE} && break
sleep 1
done
# By default, use flannel for container network plugin, should make this configurable.
sudo kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f https://raw.githubusercontent.com/coreos/flannel/v0.10.0/Documentation/kube-flannel.yml
echo done.
) 2>&1 | sudo tee /var/log/startup.log
shutdownScript: |
set -e
set -x
(
ARCH=amd64
#TODO do we drain? or require the operator of the machine to drain before updating? a node has no permission to do so internally
# sudo kubectl --kubeconfig=/etc/kubernetes/kubelet.conf drain $(hostname) --delete-local-data --ignore-daemonsets --force
sudo kubectl --kubeconfig=/etc/kubernetes/kubelet.conf delete node $(hostname)
sudo kubeadm reset
# !? TODO: This is an incomplete list of packages to remove
DEBIAN_FRONTEND=noninteractive
sudo yum erase -y kubeadm kubectl kubelet kubernetes-cni kube* docker.io
RM_RF_DIRS="/etc/cni \
/etc/docker \
/etc/ethertypes \
/etc/kubernetes \
/etc/systemd/system/kubelet.service.d \
/var/lib/cni \
/var/lib/docker \
/var/lib/dockershim \
/var/lib/etcd \
/var/lib/etcd2 \
/var/lib/kubelet"
for d in ${RM_RF_DIRS}; do
sudo rm -rf $d
done
) 2>&1 | sudo tee /var/log/teardown.log
shutdownScript: |
set -e
set -x
(
ARCH=amd64
#TODO do we drain? or require the operator of the machine to drain before updating? a node has no permission to do so internally
# sudo kubectl --kubeconfig=/etc/kubernetes/kubelet.conf drain $(hostname) --delete-local-data --ignore-daemonsets --force
sudo kubectl --kubeconfig=/etc/kubernetes/kubelet.conf delete node $(hostname)
sudo kubeadm reset
# !? TODO: This is an incomplete list of packages to remove
DEBIAN_FRONTEND=noninteractive
sudo yum erase -y kubeadm kubectl kubelet kubernetes-cni kube* docker.io
RM_RF_DIRS="/etc/cni \
/etc/docker \
/etc/ethertypes \
/etc/kubernetes \
/etc/systemd/system/kubelet.service.d \
/var/lib/cni \
/var/lib/docker \
/var/lib/dockershim \
/var/lib/etcd \
/var/lib/etcd2 \
/var/lib/kubelet"
for d in ${RM_RF_DIRS}; do
sudo rm -rf $d
done
) 2>&1 | sudo tee /var/log/teardown.log
- machineParams:
roles:
- Node
versions:
kubelet: 1.10.6
metadata:
startupScript: |
set -e
set -x
(
ARCH=amd64
function add_kubernetes_repo () {
sudo cp /dev/stdin /etc/yum.repos.d/kubernetes.repo <<< "
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
"
}
function prips () {
cidr=$1
# range is bounded by network (-n) & broadcast (-b) addresses.
lo=$(ipcalc -n $cidr |cut -f2 -d=)
hi=$(ipcalc -b $cidr |cut -f2 -d=)
read a b c d <<< $(echo $lo |tr . ' ')
read e f g h <<< $(echo $hi |tr . ' ')
eval "echo {$a..$e}.{$b..$f}.{$c..$g}.{$d..$h}"
}
# kubeadm uses 10th IP as DNS server
CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | awk '{print $11}')
KUBELET=$(kubelet-${KUBELET_VERSION})
KUBEADM=$(kubeadm-${KUBELET_VERSION})
add_kubernetes_repo
sudo yum install -y \
docker \
${KUBELET} \
${KUBEADM}
sudo cp /dev/stdin /etc/sysconfig/docker <<< 'DOCKER_OPTS="--iptables=false --ip-masq=false"'
sudo cp /dev/stdin /etc/systemd/system/kubelet.service.d/20-kubenet.conf <<< '[Service]
Environment="KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-domain=${CLUSTER_DNS_DOMAIN}"
'
# Not yet selinux ready
sudo setenforce 0
sudo systemctl daemon-reload
sudo systemctl disable --now firewalld
sudo systemctl enable --now docker kubelet
# Set up kubeadm config file to pass parameters to kubeadm init.
# Note the latest API version for kubeadm is v1alphav2 as of kubeadm
# 1.11. We need to account for these different versions in the next
# rewrite of this script.
sudo cp /dev/stdin /etc/kubernetes/kubeadm_config.yaml <<< "
apiVersion: kubeadm.k8s.io/v1alpha1
kind: MasterConfiguration
api:
advertiseAddress: ${MASTER_IP}
bindPort: 443
etcd:
local:
dataDir: /var/lib/etcd
image:
kubernetesVersion: v${CONTROL_PLANE_VERSION}
token: ${TOKEN}
kubeProxy:
config:
clusterCIDR: ${POD_CIDR}
networking:
dsnDomain: ${CLUSTER_DNS_DOMAIN}
serviceSubnet: ${SERVICE_CIDR}
podSubnet: ${POD_CIDR}
"
# Create and set bridge-nf-call-iptables to 1 to pass the kubeadm preflight check.
# Workaround was found here:
# http://zeeshanali.com/sysadmin/fixed-sysctl-cannot-stat-procsysnetbridgebridge-nf-call-iptables/
sudo modprobe br_netfilter
# [ERROR Swap]: running with swap on is not supported. Please disable swap
# this was put in place since its something that must be done on ubuntu machines
# when provisioning for use with kubeadm. Note, kubelet requires this to be off
# and may be something we remove in the future and leave it as part of provisioning
sudo swapoff -a
sudo kubeadm join --token "${TOKEN}" "${MASTER}" --ignore-preflight-errors=all --discovery-token-unsafe-skip-ca-verification
for tries in $(seq 1 60); do
sudo kubectl --kubeconfig /etc/kubernetes/kubelet.conf annotate --overwrite node $(hostname) machine=${MACHINE} && break
sleep 1
done
echo done.
) 2>&1 | sudo tee /var/log/startup.log
shutdownScript: |
set -e
set -x
(
ARCH=amd64
#TODO do we drain? or require the operator of the machine to drain before updating? a node has no permission to do so internally
# sudo kubectl --kubeconfig=/etc/kubernetes/kubelet.conf drain $(hostname) --delete-local-data --ignore-daemonsets --force
sudo kubectl --kubeconfig=/etc/kubernetes/kubelet.conf delete node $(hostname)
sudo kubeadm reset
# !? TODO: This is an incomplete list of packages to remove
DEBIAN_FRONTEND=noninteractive
sudo apt-get purge kubeadm kubectl kubelet kubernetes-cni kube* docker.io -y
RM_RF_DIRS="/etc/cni \
/etc/docker \
/etc/ethertypes \
/etc/kubernetes \
/etc/systemd/system/kubelet.service.d \
/var/lib/cni \
/var/lib/docker \
/var/lib/dockershim \
/var/lib/etcd \
/var/lib/etcd2 \
/var/lib/kubelet"
for d in ${RM_RF_DIRS}; do
sudo rm -rf $d
done
) 2>&1 | sudo tee /var/log/teardown.log
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment