쿠버네티스 설치부터 helm 을 이용한 istio service mesh 구성까지

[Ubuntu] Docker, Kubernetes, helm Install Shell Script

#!/bin/bash

RED=`tput setaf 1`
GREEN=`tput setaf 2`
NC=`tput sgr0`

# Check permission
if [ "$EUID" -ne 0 ]
  then echo "${RED}Please run as root ${NC}"
  exit
fi

############ language change ###############
echo "${RED}LANGUAGE CHANGE${NC}"
localedef -c -i ko_KR -f UTF-8 ko_KR.utf8
localectl set-locale LANG=ko_KR.utf8

############### Timezone ###################
echo "${RED}TIMEZONE CHANGE${NC}"
timedatectl set-timezone Asia/Seoul
echo '[Timezone] Change Success'

############ hostname change ###############
echo "${RED}HOSTNAME CHANGE${NC}"
read -p "hostname Change is (ex k8s-worker) :" uhost
hostnamectl set-hostname $uhost
echo '[Hostname] Change Success'

############ mirror change ###############
echo "${RED}APT MIRROR CHANGE${NC}"
sed -i 's/nova.clouds.archive.ubuntu.com/mirror.kakao.com/g' /etc/apt/sources.list
echo '[Mirror] Change Success'

############ Init Update ###############
apt-get update -y
apt-get install vim apt-transport-https gnupg2 curl -y


############ Docker Install ###############
echo "${RED}DOCKER INSTALL${NC}"
apt-get install docker.io -y
systemctl enable --now docker
docker --version
echo '[Docker] Success'

############ K8s & Helm Install ###############
echo "${RED}K8S & HELM INSTALL${NC}"
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
curl https://helm.baltorepo.com/organization/signing.asc | sudo apt-key add -
echo "deb https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list
apt-get update -y
apt-get install kubelet kubeadm kubectl helm -y
apt-mark hold kubelet kubeadm kubectl
echo '[k8s, Helm] Success'

############ Swap off ###############
echo "${RED}SWAP OFF${NC}"
swapoff -a && sudo sed -i.bak 's/\/swap\.img/#\/swap\.img/g' /etc/fstab
echo '[Swap Off] Success'

############ Kernel Network Edit ###############
echo "${RED}SET IP FORWARDING${NC}"
echo 1 > /proc/sys/net/ipv4/ip_forward
# sysctl -w net.ipv4.ip_forward=1

echo "${RED}SET NETWORK CONFIGURATION${NC}"
modprobe br_netfilter
cat <<EOF > /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF
sysctl --system 
echo '[Kernel Netowrk Edit] Success'

systemctl daemon-reload
systemctl restart kubelet
systemctl enable kubelet

########### Docker daemon Added ##############
echo "${RED}DOCKER DAEMON EDIT${NC}"
cat <<EOF > /etc/docker/daemon.json
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2"
}
EOF
mkdir -p /etc/systemd/system/docker.service.d
systemctl daemon-reload
systemctl restart docker

############ Firewalls with UFW Added ###############
echo "${RED}FIREWALLD RULES UPDATED${NC}"
ufw allow ssh
ufw allow 6443
ufw allow 2379
ufw allow 2380
ufw allow 10250
ufw allow 10251
ufw allow 10252

############ Init and Network Setting ###############
echo "${RED}K8S INSTALLING${NC}"
my_vm_internal_ip="$(hostname -I | awk {'print $1'})"
flannel_cidr="10.244.0.0/16"
echo '#### K8s Init ? ####'
echo '[Kubernetes Init Select]'
echo 'Network Add-on is [Flannel]'
echo 'Flannel Applying...'
kubeadm init --pod-network-cidr=${flannel_cidr} --apiserver-advertise-address=${my_vm_internal_ip}
echo ' '
echo '######## Token 복사하시고, Cluster Node에 붙이세요. ############'
echo ' '
mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config
#export KUBECONFIG=/etc/kubernetes/admin.conf
echo "${RED}NETWORK - FLANNEL INSTALLING${NC}"
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
echo '[flannel] Success'

########### Example Domains ###############
cat <<EOF >> /etc/hosts
${my_vm_internal_ip} ${uhost}
EOF

########### Option #############
# kubectl taint nodes --all node-role.kubernetes.io/master-

master 노드에서 worker 노드로의 접속

Local 에서 Master 노드로 키 전송

scp -i taking.pem taking.pem [email protected]:/home/ubuntu/

ubuntu 계정에서 ssh 키 설정

mv taing.pem ~/.ssh
chmod 600 ~/.ssh/taking.pem
chmod 700 ~/.ssh/
ssh -i ~/.ssh/taking.pem [email protected]

Metrics-Server 설치

• OO

kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
apt install jq -y

docker 초기화

docker stop $(docker ps -a -q)  &&  docker rmi $(docker images -q)

k3s

#!/bin/bash

apt update
apt upgrade

cluster_cidr="172.20.0.0/16"
service_cidr="172.24.0.0/16"

curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --cluster-cidr ${cluster_cidr} --service-cidr ${service_cidr} --cluster-dns ${cluster_cidr}  --no-deploy \"servicelb\" --no-deploy \"metrics-server\" --no-deploy \"local-storage\"" sh -
kubectl get nodes

echo 'config file path = /etc/rancher/k3s/k3s.yaml'

• k3s 의 콘픽은 /etc/rancher/k3s/k3s.yaml 에 있음
• 당연히 외부에서 접근하려면 해당 yml 파일의 server 의 127.0.0.1 부분을 public IP로 변경 필요

curl -sfL https://get.k3s.io | K3S_URL=https://serverip:6443 K3S_TOKEN=mytoken sh - 를 이용하여 Node 추가

• servicelb 를 설치 시, 제외 했기 떄문에 필요 시 metallb 추가 설치 필요

kubectl config set-context default --cluster='cluster-1' &&
kubectl config set-context default --user='cluster-1' &&
kubectl config rename-context default cluster-1 &&
sed -i 's/  name: default/  name: cluster-1/g' /etc/rancher/k3s/k3s.yaml &&
sed -i 's/- name: default/- name: cluster-1/g' /etc/rancher/k3s/k3s.yaml &&
kubectl get nodes

kubectl -n kube-system create serviceaccount cluster-1
kubectl create clusterrolebinding cluster-1\
  --clusterrole=cluster-admin \
  --serviceaccount=kube-system:cluster-1

• 외부에서 접속을 위해선 k3s 의 --tls-san 옵션에 할당된 public IP를 입력해줘야 함
  
• 그리고 서비스 재시작

systemctl daemon-reload
systemctl restart k3s

metallb

• 위에 잠깐 나왔지만, 이어서 보기 편하도록 추가

kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.9.5/manifests/namespace.yaml
kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.9.5/manifests/metallb.yaml
kubectl create secret generic -n metallb-system memberlist \
  --from-literal=secretkey='$(openssl rand -base64 128)'

• 아래 addresses 부분을 cluster ip 로 변경하여 진행

cat <<EOF > ~/metallb-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: metallb-system
  name: config
data:
  config: |
    address-pools:
    - name: default
      protocol: layer2
      addresses:
      - 172.20.0.0/16
EOF
kubectl apply -f ~/metallb-configmap.yaml

• public ip 할당 받을 수 있는 경우, 아래와 같이 가능

addresses: 
- 192.168.100.100-192.168.100.250