HW3_k8s_CEPHFS.sh

#!/bin/bash
NC='\033[0m' # No Color
YELLOW='\033[1;33m'

cephfs_ip="10.0.2.15" # ceph mon dump
clusterID="e952c52a-f18f-11ec-bb41-08002716126b" # ceph mon dump
cephfs_user_key="AQAfDrJibQtyARAAp3DxKWUP2LBCPbJUe9BUDg==" # ceph auth get-or-create client.cephfs mon 'allow r' osd 'allow rwx pool=kubernetes'
cephfs_admin_key="AQDzDbJiuBawARAA2NBP5/wgoaadOYGWIrUFFA==" # ceph auth get client.admin

sudo dnf update -y &&\
sudo dnf install -y kernel-devel kernel-header* make gcc elfutils-libelf-devel

printf "${YELLOW}Installing Docker${NC}\n"
sudo yum install -y yum-utils telnet &&\
sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo &&\
sudo yum install -y --allowerasing git docker-ce docker-ce-cli containerd.io docker-compose-plugin &&\
sudo systemctl enable docker.service &&\
sudo systemctl start docker.service &&\
printf "${YELLOW}Docker install complete${NC}\n"

# https://docs.docker.com/engine/install/linux-postinstall/
printf "${YELLOW}Get Permissions as recommended\n${NC}"
sudo usermod -aG docker ${USER}
sudo chmod 666 /var/run/docker.sock # Due to permission problem. I can't get GROUP(docker) permission with Account of User, named 'test'

printf "${YELLOW}Open Port${NC}\n"
# sudo firewall-cmd --zone=public --permanent --add-port 443/tcp
# sudo firewall-cmd --zone=public --permanent --add-port 6443/tcp
# sudo firewall-cmd --zone=public --permanent --add-port 2379/tcp
# sudo firewall-cmd --zone=public --permanent --add-port 2380/tcp
# sudo firewall-cmd --zone=public --permanent --add-port 8080/tcp
# sudo firewall-cmd --zone=public --permanent --add-port 10250/tcp
# sudo firewall-cmd --zone=public --permanent --add-port 10259/tcp
# sudo firewall-cmd --zone=public --permanent --add-port 10257/tcp
# sudo firewall-cmd --zone=public --permanent --add-port 179/tcp # Calico networking (BGP)
# sudo firewall-cmd --zone=public --permanent --add-port 3300/tcp # CEPH
# sudo firewall-cmd --zone=public --permanent --add-port 6789/tcp # CEPH
# sudo firewall-cmd --reload

sudo systemctl stop firewalld
sudo systemctl disable firewalld
sudo systemctl mask --now firewalld

printf "${YELLOW}Check br_netfilter${NC}\n"
sudo printf "br_netfilter" > /etc/modules-load.d/k8s.conf
sudo printf "net.bridge.bridge-nf-call-ip6tables = 1\nnet.bridge.bridge-nf-call-iptables = 1" > /etc/sysctl.d/k8s.conf
sudo sysctl --system

printf "${YELLOW}SELinux permissive${NC}\n"
sudo setenforce 0
sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config

printf "${YELLOW}Swap off${NC}\n"
sudo swapon && sudo cat /etc/fstab
sudo swapoff -a && sudo sed -i '/swap/s/^/#/' /etc/fstab

printf "${YELLOW}Install Kubernetes${NC}\n"
sudo cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
EOF
sudo yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes
sudo systemctl enable --now kubelet && sudo systemctl restart kubelet

printf "${YELLOW}Configure cgroup driver${NC}\n"
sudo mkdir -p /etc/docker
sudo cat <<EOF | sudo tee /etc/docker/daemon.json
{
    "exec-opts": ["native.cgroupdriver=systemd"],
    "log-driver": "json-file",
    "log-opts": {
      "max-size": "100m"
    },
    "storage-driver": "overlay2"
}
EOF
sudo systemctl enable docker
sudo systemctl daemon-reload
sudo systemctl restart docker

# Due to Error: "getting status of runtime: rpc error: code = Unimplemented desc = unknown service runtime.v1alpha2.RuntimeService"
sudo rm -f /etc/containerd/config.toml
sudo systemctl restart containerd
sudo kubeadm config images pull
sudo kubeadm init

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

# CNI - Install Weave
kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')"

# Install Calico
# sudo cat <<EOF | sudo tee ./custom-resources.yaml
# apiVersion: operator.tigera.io/v1
# kind: Installation
# metadata:
#   name: default
# spec:
#   calicoNetwork:
#     ipPools:
#     - blockSize: 26
#       cidr: 10.0.0.0/8
#       encapsulation: VXLANCrossSubnet
#       natOutgoing: Enabled
#       nodeSelector: all()
# 
# ---
# apiVersion: operator.tigera.io/v1
# kind: APIServer 
# metadata: 
#   name: default 
# spec: {}
# EOF

# kubectl create -f https://projectcalico.docs.tigera.io/manifests/tigera-operator.yaml
# kubectl create -f ./custom-resources.yaml
kubectl taint nodes --all node-role.kubernetes.io/master-
kubectl taint nodes --all node-role.kubernetes.io/control-plane-

# https://github.com/kubernetes-csi/external-snapshotter#usage
# Install Snapshot CRDs:
# cd ~/
# git clone https://github.com/kubernetes-csi/external-snapshotter.git
# kubectl kustomize ~/external-snapshotter/client/config/crd | kubectl create -f -
# Install Common Snapshot Controller
## kubectl -n default kustomize ~/external-snapshotter/deploy/kubernetes/snapshot-controller | kubectl create -f -
# Install CSI Driver:
## kubectl kustomize ~/external-snapshotter/deploy/kubernetes/csi-snapshotter | kubectl create -f -

# GA(General Availiable): >= 1.20)
# Enable Feature Gate: VolumeSnapshotDataSource
# sudo sed -i'' -r -e "/    - kube-apiserver/a\    - --feature-gates=VolumeSnapshotDataSource=true/" /etc/kubernetes/manifests/kube-apiserver.yaml
# sudo sed -i'' -r -e "/    - kube-controller-manager/a\    - --feature-gates=VolumeSnapshotDataSource=true/" /etc/kubernetes/manifests/kube-controller-manager.yaml
# sudo sed -i'' -r -e "/    - kube-scheduler/a\    - --feature-gates=VolumeSnapshotDataSource=true/" /etc/kubernetes/manifests/kube-scheduler.yaml

# New Trial
git clone https://github.com/ceph/ceph-csi.git
kubectl apply -f ./ceph-csi/examples/ceph-conf.yaml
sudo cat <<EOF | sudo tee ./ceph-csi/deploy/cephfs/kubernetes/csi-config-map.yaml
---
apiVersion: v1
kind: ConfigMap
data:
  config.json: |-
    [
      {
        "clusterID": "${clusterID}",
        "monitors": [
          "${cephfs_ip}:6789",
          "${cephfs_ip}:3300"
        ],
        "cephFS": {
          "subvolumeGroup": "kubernetes"
        }
      }  
    ]
metadata:
  name: ceph-csi-config
EOF

sudo cat <<EOF | sudo tee  ./ceph-csi/deploy/cephfs/kubernetes/secret.yaml
---
apiVersion: v1
kind: Secret
metadata:
  name: csi-cephfs-secret
  namespace: default
stringData:
  # Required for statically provisioned volumes
  userID: cephfs
  userKey: ${cephfs_user_key}

  # Required for dynamically provisioned volumes
  adminID: admin
  adminKey: ${cephfs_admin_key}
EOF

sudo cat <<EOF | sudo tee ./ceph-csi/deploy/cephfs/kubernetes/storageclass.yaml
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: csi-cephfs-sc
provisioner: cephfs.csi.ceph.com
parameters:
  clusterID: ${clusterID}
  fsName: kubernetes

  csi.storage.k8s.io/provisioner-secret-name: csi-cephfs-secret
  csi.storage.k8s.io/provisioner-secret-namespace: default
  csi.storage.k8s.io/controller-expand-secret-name: csi-cephfs-secret
  csi.storage.k8s.io/controller-expand-secret-namespace: default
  csi.storage.k8s.io/node-stage-secret-name: csi-cephfs-secret
  csi.storage.k8s.io/node-stage-secret-namespace: default

reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:
  - debug
EOF

sudo cat <<EOF | sudo tee ./ceph-csi/deploy/cephfs/kubernetes/pvc.yaml
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: csi-cephfs-pvc
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Gi
  storageClassName: csi-cephfs-sc
EOF

sudo cat <<EOF | sudo tee ./ceph-csi/deploy/cephfs/kubernetes/pod.yaml
---
apiVersion: v1
kind: Pod
metadata:
  name: csi-cephfs-demo-pod
spec:
  containers:
    - name: web-server
      image: docker.io/library/nginx:latest
      volumeMounts:
        - name: mypvc
          mountPath: /var/lib/www
  volumes:
    - name: mypvc
      persistentVolumeClaim:
        claimName: csi-cephfs-pvc
        readOnly: false
EOF

sudo cat <<EOF | sudo tee ./ceph-csi/deploy/cephfs/kubernetes/snapshotclass.yaml
---
apiVersion: snapshot.storage.k8s.io/v1
kind: VolumeSnapshotClass
metadata:
  name: csi-cephfsplugin-snapclass
driver: cephfs.csi.ceph.com
parameters:
  clusterID: ${clusterID}
  csi.storage.k8s.io/snapshotter-secret-name: csi-cephfs-secret
  csi.storage.k8s.io/snapshotter-secret-namespace: default
deletionPolicy: Delete
EOF

cd $HOME/ceph-csi/examples/cephfs/
sh plugin-deploy.sh

cd $HOME/ceph-csi/deploy/cephfs/kubernetes/
kubectl create -f csidriver.yaml &&\        # Need for pod.yaml
kubectl create -f secret.yaml &&\
kubectl create -f storageclass.yaml &&\
kubectl create -f pvc.yaml &&\
kubectl create -f pod.yaml

# Reference: https://yjwang.tistory.com/125

git clone https://github.com/kubernetes-csi/external-snapshotter.git
kubectl kustomize ~/external-snapshotter/client/config/crd | kubectl create -f -
kubectl create -f ~/ceph-csi/deploy/cephfs/kubernetes/snapshotclass.yaml
watch kubectl get all

cd ~
# kubectl create -f ./ceph-csi/examples/cephfs/snapshot.yaml # when need snapshot

다 됐다고 생각했는데, PVC가 안 만들어짐

Calico -> Weave 로 CNI를 바꾸니 PVC가 정상적으로 생김 (PV가 알아서 생겼음)
아마 CIDR 설정에서 (10.0.0.0/8) 이 정답이 아닌거같은데,
어떻게 해결해야할지는 네트워크에 대해서 좀더 생각해봐야할 것 같음.

• 어제는 Weave가 몇 시간동안 설치가 안되던데, 일시적인 장애였나?

CEPH CSI 설명에는 안 나와있지만, ./ceph-csi/deploy/cephfs/kubernetes/csidriver.yaml 를 apply 해줘야하는 것 같음. (하니깐 demo pod이 작동함)
kubectl apply -f ceph-csi/deploy/cephfs/kubernetes/csidriver.yaml

Snapshot도 정상적으로 작동되는 것으로 보임.