Install Kubernetes on an Offline CentOS 7 Machine - (2) OFFLINE SERVER

pkg
├── 1-server
├── 2-cri
├── 3-k8s
├── 4-cni
├── 5-gpu
└── 6-mec

Configure network

systemctl is-enabled NetworkManager
# nmtui

# /etc/hosts
192.168.213.160 host01

# /etc/sysconfig/network-scripts/ifcfg-eth0
# /etc/sysconfig/network-scripts/ifcfg-eno1
...
IPADDR=192.168.213.160
PREFIX=24 # or NETMASK=255.255.255.0
NETWORK=192.168.213.0
GATEWAY=192.168.213.1
BROADCAST=192.168.213.255
...

# DNS
cat << EOF > /etc/resolv.conf
nameserver 8.8.8.8
EOF

systemctl restart network
systemctl is-enabled network

Copy files from flash drive

# Mount Drive
mkdir -p /media/usb
ls -halt /dev/sd*

# Check the information and mount it
mount -t vfat /dev/sdb1 /media/usb
ls -halt /media/usb

# Copy files
cp -rv /media/usb/pkg/ $HOME
ls -halt $HOME/pkg

# Unmount Drive
umount /media/usb
ls -halt /media/usb

Kubespray

# rpm -ivh --replacefiles --replacepkgs $HOME/pkg/kubespray/python/*.rpm

# pip3 install $HOME/pkg/kubespray/deps/*

Install Docker

cd $HOME/pkg/2-cri
tar -zxvf $HOME/pkg/2-cri/docker.tar.gz -C $HOME/pkg/2-cri
rpm -ivh --replacefiles --replacepkgs *.rpm
systemctl enable docker
systemctl start docker
docker ps

cat > /etc/docker/daemon.json << EOF
{
  "insecure-registries": [
    "mec03:5000",
    "192.168.213.4:5000",
    "127.0.0.1:5000"
  ]
}
EOF

systemctl restart docker

Set Kernel parameter as required by Kubernetes (K8s).

cat > /etc/sysctl.d/kubernetes.conf << EOF
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

Reload Kernel parameter configuration files.

modprobe br_netfilter
sysctl --system
# Applying /etc/sysctl.d/kubernetes.conf ...

Turn off Swap for Kubernetes (K8s) installation.

swapoff -a
sed -e '/swap/s/^/#/g' -i /etc/fstab

Switch SELinux to Permissive mode using following commands.

setenforce 0
sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config

firewalld is active, please ensure ports [6443 10250] are open or your cluster may not function correctly

# firewall-cmd --permanent --add-port=6443/tcp
# firewall-cmd --permanent --add-port=10250/tcp
# firewall-cmd --reload
systemctl stop firewalld
systemctl disable firewalld
systemctl is-enabled firewalld

tar -xvf $HOME/pkg/1-server/server.tar.gz -C $HOME/pkg/1-server
rpm -ivh --replacefiles --replacepkgs *.rpm
rm -f *.rpm

docker load -i $HOME/pkg/1-server/nfs-provisioner.tar
docker load -i $HOME/pkg/1-server/redis.tar
docker load -i $HOME/pkg/1-server/kube-ops-view.tar
docker load -i $HOME/pkg/1-server/grafana.tar
docker load -i $HOME/pkg/1-server/kube-state-metrics.tar
docker load -i $HOME/pkg/1-server/node-exporter.tar
docker load -i $HOME/pkg/1-server/alertmanager.tar
docker load -i $HOME/pkg/1-server/ghostunnel.tar
docker load -i $HOME/pkg/1-server/kube-webhook-certgen.tar
docker load -i $HOME/pkg/1-server/prometheus-operator.tar
docker load -i $HOME/pkg/1-server/configmap-reload.tar
docker load -i $HOME/pkg/1-server/prometheus-config-reloader.tar
docker load -i $HOME/pkg/1-server/prometheus.tar

# chrony
systemctl status chronyd
timedatectl status | grep -i time

# ssh
systemctl status sshd
ssh-keygen -t rsa
ssh-copy-id root@host01
ssh-copy-id root@host02
ssh-copy-id root@host03

# nfs-server
# worker node (nfs-server)
systemctl status nfs-server
systemctl stop nfs-server
df -h # Mounted on 용량이 가장 큰 /home에 지정
mkdir /home/mec-data
cat << EOF > /etc/exports
/home/mec-data 192.168.213.* (rw,all_squash,sync)
EOF
systemctl start nfs-server
systemctl enable nfs-server
systemctl status nfs-server

# master node (nfs-client)
mkdir /home/mec-data
mount -t nfs mec01:/home/gigamec-data /home/gigamec-data/
# umount --force /home/mec-data
# umount --lazy /home/mec-data
systemctl start nfs-server
systemctl enable nfs-server
systemctl status nfs-server
# > Created symlink from /etc/systemd/system/multi-user.target.wants/nfs-server.service to /usr/lib/systemd/system/nfs-server.service.
exportfs -rav

# $ showmount --exports
# > Export list for host01:
# $ showmount --exports 192.168.213.221
# > Export list for 192.168.213.221:
# > /home/mec-data (everyone)
# $ cat /proc/mounts
# $ df -h | grep mec-data
# > host01:/home/mec-data  180G  32M  180G   1% /home/mec-data
# $ mount -v host01

Installing Kubernetes (K8s) Offline on CentOS 7

tar -xvf $HOME/pkg/3-k8s/kubernetes.tar.gz -C $HOME/pkg/3-k8s
rpm -ivh --replacefiles --replacepkgs $HOME/pkg/3-k8s/*.rpm
rm -f *.rpm
source <(kubectl completion bash)
kubectl completion bash > /etc/bash_completion.d/kubectl

docker load < $HOME/pkg/3-k8s/coredns.tar
docker load < $HOME/pkg/3-k8s/kube-proxy.tar
docker load < $HOME/pkg/3-k8s/etcd.tar
docker load < $HOME/pkg/3-k8s/kube-scheduler.tar
docker load < $HOME/pkg/3-k8s/kube-apiserver.tar
docker load < $HOME/pkg/3-k8s/pause.tar
docker load < $HOME/pkg/3-k8s/kube-controller-manager.tar

w/o ansible

# ansible-playbook -i inventory/mec03/inventory.ini --become --become-user=root cluster.yml -vvvv

# Check route rule
# cat /proc/net/route
route
# default         gateway         0.0.0.0         UG    100    0        0 eno1
# 172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0
# 192.168.213.0   0.0.0.0         255.255.255.0   U     100    0        0 eno1

# INIT
# A Class: 10.0.0.0/8
# B Class: 172.16.0.0/12
# C Class: 192.168.0.0/16
kubeadm config images list

# https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2#ClusterConfiguration
cat > $HOME/pkg/kubeadm-config.yaml << EOF
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
kubernetesVersion: "v1.18.2"
networking:
  podSubnet: "10.233.0.0/18"
  serviceSubnet: "10.233.64.0/18"
apiServer:
  extraArgs:
    advertise-address: "192.168.213.4"
controlPlaneEndpoint: "192.168.213.4:6443" 
clusterName: "cluster.name"
EOF

# kubeadm init --service-cidr=10.233.0.0/18 --pod-network-cidr=10.233.64.0/18 --apiserver-advertise-address=192.168.213.160 --kubernetes-version=1.18.2 --v=5
kubeadm init --config=$HOME/pkg/kubeadm-config.yaml --upload-certs --v=5

# kubelet-check timeout
systemctl status kubelet
journalctl -u kubelet -ex | less
# journalctl -u kubelet -fx | less
# Failed to get status for pod ~: Get ~: dial tcp ~: connect: network is unreachable
# Failed to set some node status fields: can't get ip address of node host01. error: no default routes found in "/proc/net/route" or "/proc/net/ipv6_route"
# Unable to register node "host01" with API server: Post 192.168.1.1:6443/api/v1/nodes:dial tcp 192.168.1.1:6443: connect: network is unreachable
# 랜선은 꽂자...

# Execute suggested script.
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

# Start and enable kubelet.service.
# systemctl enable kubelet
# systemctl start kubelet

Install Calico for CNI

cp -v $HOME/pkg/4-cni/calico /opt/cni/bin/calico
cp -v $HOME/pkg/4-cni/calico-ipam /opt/cni/bin/calico-ipam
cp -rv $HOME/pkg/4-cni/etc-cni-net.d/ /etc/cni/net.d/

docker load < $HOME/pkg/4-cni/cni.tar
docker load < $HOME/pkg/4-cni/pod2daemon-flexvol.tar
docker load < $HOME/pkg/4-cni/node.tar
docker load < $HOME/pkg/4-cni/kube-controllers.tar

kubectl apply -f $HOME/pkg/4-cni/calico-etcd.yaml

List nodes in Kubernetes (K8s) cluster.

kubectl get nodes

3. Kubernetes worker node or a control-plane node and adds it to the cluster.

# master node
kubeadm token list

# discovery-token-ca-cert-hash
openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | \
openssl dgst -sha256 -hex | sed 's/^.* //'

# join any number of the control-plane node
# control-plane-certificate-key
# As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use "kubeadm init phase upload-certs --upload-certs" to reload certs afterward.
kubeadm init phase upload-certs --upload-certs

# list nodes
kubectl --kubeconfig /etc/kubernetes/admin.conf get nodes

# worker node
kubeadm join 192.168.7.221:6443 --token <token-name> \
--discovery-token-ca-cert-hash sha256:<discovery-token-ca-cert-hash>
# --control-plane --certificate-key <control-plane-certificate-key> \

Applications

mkdir $HOME/images

docker load -i $HOME/pkg/2-cri/registry.tar

docker run \
--detach \
--publish 5000:5000 \
--restart=always \
--name registry \
--env REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY=$HOME/images \
--env REGISTRY_STORAGE_DELETE_ENABLED=true \
registry:2.7.1

# docker push repo:5000/app:tag

tar -zxvf $HOME/pkg/3-k8s/helm.tar.gz
mv $HOME/pkg/3-k8s/linux-amd64/helm /usr/local/bin/helm
rm -rf $HOME/pkg/3-k8s/linux-amd64/

mkdir $HOME/pkg/6-mec/test
tar -zxvf $HOME/pkg/6-mec/test.tar.gz -C $HOME/pkg/6-mec/

# replace hostname
# [command]

# grep -ir old
grep -ir old-name

# grep -rl "old" * | xargs sed -i 's/old/new/g'
# grep -rl "old-name" * | xargs sed -i 's/old-name/new-name/g'

# find ./ -name "*.yaml" -exec sed -i 's/old/new/g' {} \;
find ./ -name "*.yaml" -exec sed -i 's/old-name/new-name/g' {} \;

# [vim]
# :%s/old/new/g

# 노드가 하나일 경우
# kubectl taint nodes --all node-role.kubernetes.io/master-
# helm install <name> <chart_path> --namespace <namespace>

markruler/offline-k8s-guide-2.md

Install Kubernetes on an Offline CentOS 7 Machine - (2) OFFLINE SERVER

w/o ansible

3. Kubernetes worker node or a control-plane node and adds it to the cluster.

Applications