User Data Installation for EC2 RKE2 (CNI Calico)

rancher_rke2_addons.sh

#!/bin/bash
# ============================================================================== #
#  Configuration for Docker runtime
# ============================================================================== #
echo "[INFO] Installing and configuring Docker..."

if [[ "$OS" == "Ubuntu" ]]; then
    $PKG_INSTALL ca-certificates gnupg lsb-release
    mkdir -p /etc/apt/keyrings
    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
    echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
    $PKG_MANAGER update
    $PKG_INSTALL docker-ce docker-ce-cli containerd.io
    usermod -aG docker ubuntu
else
    $PKG_INSTALL docker
    usermod -aG docker ec2-user
fi

cat > /etc/docker/daemon.json <<EOF
{
    "exec-opts": ["native.cgroupdriver=systemd"],
    "log-driver": "json-file",
    "log-opts": {
        "max-size": "100m",
        "max-file": "3"
    },
    "storage-driver": "overlay2"
}
EOF

systemctl daemon-reload
systemctl enable docker
systemctl start docker

# Install Docker Compose v2
mkdir -p /usr/local/lib/docker/cli-plugins/
curl -SL https://github.com/docker/compose/releases/download/v2.23.3/docker-compose-linux-x86_64 -o /usr/local/lib/docker/cli-plugins/docker-compose
chmod +x /usr/local/lib/docker/cli-plugins/docker-compose


echo "[INFO] Installing additional development tools..."
# ============================================================================== #
#  Install kubectl
# ============================================================================== #
echo "[INFO] Installing kubectl..."
curl -LO "https://dl.k8s.io/release/v1.30.0/bin/linux/amd64/kubectl"
chmod +x kubectl
mv kubectl /usr/local/bin/

# ============================================================================== #
#  Install Helm
# ============================================================================== #
echo "[INFO] Installing Helm..."
curl -fsSL https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash

# ============================================================================== #
#  Install k9s
# ============================================================================== #
K9S_VERSION="v0.32.7"
curl -LO "https://github.com/derailed/k9s/releases/download/$K9S_VERSION/k9s_Linux_amd64.tar.gz"
tar -xzf k9s_Linux_amd64.tar.gz
chmod +x k9s
mv k9s /usr/local/bin/
rm -f k9s_Linux_amd64.tar.gz LICENSE README.md


# ============================================================================== #
#  Install Rancher RKE2
# ============================================================================== #
# Prerequisites Check
echo "[INFO] Checking prerequisites..."

# Check if running as root
if [[ $EUID -ne 0 ]]; then
   echo "[ERROR] This script must be run as root"
   exit 1
fi

# Check if RKE2 is installed
if ! command -v rke2 >/dev/null 2>&1; then
    echo "[ERROR] RKE2 is not installed"
    exit 1
fi

# Install required packages
echo "[INFO] Installing required packages..."
if command -v apt-get >/dev/null 2>&1; then
    apt-get update
    apt-get install -y curl jq
elif command -v yum >/dev/null 2>&1; then
    yum install -y curl jq
fi

# Configure RKE2 for Calico
echo "[INFO] Configuring RKE2 for Calico..."
mkdir -p /etc/rancher/rke2

cat > /etc/rancher/rke2/config.yaml <<EOF
cni: calico
cluster-cidr: ${POD_CIDR}
service-cidr: ${SERVICE_CIDR}
cluster-domain: ${CLUSTER_DOMAIN}
EOF

# If this is a server node, add additional configurations
if [ "${NODE_ROLE}" = "server" ]; then
    cat >> /etc/rancher/rke2/config.yaml <<EOF
disable:
  - rke2-canal
  - rke2-multus
EOF
fi

# Create Calico configuration
echo "[INFO] Creating Calico configuration..."
mkdir -p /var/lib/rancher/rke2/server/manifests

cat > /var/lib/rancher/rke2/server/manifests/calico-installation.yaml <<EOF
---
apiVersion: operator.tigera.io/v1
kind: Installation
metadata:
  name: default
spec:
  registry: docker.io
  calicoNetwork:
    ipPools:
    - blockSize: 26
      cidr: ${POD_CIDR}
      encapsulation: ${ENABLE_VXLAN:+"VXLAN"}${ENABLE_BGP:+"IPIPCrossSubnet"}
      natOutgoing: true
      nodeSelector: all()
    nodeAddressAutodetectionV4:
      firstFound: true
EOF

if [ "${ENABLE_WIREGUARD}" = "true" ]; then
    cat >> /var/lib/rancher/rke2/server/manifests/calico-installation.yaml <<EOF
  nodeEncryption:
    enabled: true
    encryptionMethod: WireGuard
EOF
fi

# Create monitoring configuration
echo "[INFO] Setting up monitoring configuration..."
cat > /var/lib/rancher/rke2/server/manifests/calico-monitoring.yaml <<EOF
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: calico-node-monitoring
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: calico-node-monitoring
rules:
- apiGroups: [""]
  resources:
  - pods
  - nodes
  verbs:
  - get
  - list
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: calico-node-monitoring
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: calico-node-monitoring
subjects:
- kind: ServiceAccount
  name: calico-node-monitoring
  namespace: kube-system
EOF

# Create AWS integration configuration
if [ "${NODE_ROLE}" = "server" ]; then
    echo "[INFO] Setting up AWS integration..."
    cat > /var/lib/rancher/rke2/server/manifests/calico-aws.yaml <<EOF
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: aws-config
  namespace: kube-system
data:
  AWS_REGION: "${AWS_REGION}"
EOF
fi

# Setup Calico network policies
echo "[INFO] Creating default network policies..."
cat > /var/lib/rancher/rke2/server/manifests/calico-network-policies.yaml <<EOF
---
apiVersion: projectcalico.org/v3
kind: GlobalNetworkPolicy
metadata:
  name: default-deny
spec:
  selector: all()
  types:
  - Ingress
  - Egress
EOF

# Configure logging
echo "[INFO] Setting up logging configuration..."
mkdir -p /var/log/calico
cat > /etc/rsyslog.d/calico.conf <<EOF
if \$programname == 'calico' then /var/log/calico/calico.log
& stop
EOF
systemctl restart rsyslog

# Setup log rotation
cat > /etc/logrotate.d/calico <<EOF
/var/log/calico/calico.log {
    daily
    rotate 7
    compress
    delaycompress
    missingok
    notifempty
    create 640 root root
}
EOF

# Restart RKE2 service
echo "[INFO] Restarting RKE2 service..."
if [ "${NODE_ROLE}" = "server" ]; then
    systemctl restart rke2-server
else
    systemctl restart rke2-agent
fi

# Verify installation
echo "[INFO] Waiting for services to start..."
sleep 30

if [ "${NODE_ROLE}" = "server" ]; then
    # Wait for Calico to be ready
    export KUBECONFIG=/etc/rancher/rke2/rke2.yaml
    echo "[INFO] Waiting for Calico pods to be ready..."
    until kubectl get pods -n kube-system -l k8s-app=calico-node -o jsonpath='{.items[*].status.phase}' | grep -q "^Running Running"; do
        echo "Waiting for Calico pods..."
        sleep 10
    done
elif [ "$NODE_ROLE" = "observability" ]; then
    # Setup data directories with correct permissions
    mkdir -p /data/{prometheus,grafana,loki}
    chown 65534:65534 /data/prometheus
    chown 472:472 /data/grafana
    chown 10001:10001 /data/loki
fi

echo "[INFO] Installation completed successfully"
echo "[END] Calico Installation - $(date)"

# Create verification script
cat > /usr/local/bin/verify-calico <<EOF
#!/bin/bash
export KUBECONFIG=/etc/rancher/rke2/rke2.yaml
echo "Checking Calico pods status..."
kubectl get pods -n kube-system -l k8s-app=calico-node
echo "Checking Calico node status..."
kubectl get nodes -o custom-columns=NAME:.metadata.name,INTERNAL-IP:.status.addresses[0].address,STATUS:.status.conditions[4].type
echo "Checking network policies..."
kubectl get networkpolicies --all-namespaces
EOF
chmod +x /usr/local/bin/verify-calico

echo "To verify the installation, run: verify-calico"


# ============================================================================== #
#  Create instance metadata file
# ============================================================================== #
echo "[INFO] Creating instance metadata file..."
cat > /etc/instance-tags <<EOF
NODE_ROLE=$NODE_ROLE
CLUSTER_NAME=$CLUSTER_NAME
HOSTNAME=$HOSTNAME
ENVIRONMENT=$ENVIRONMENT
AWS_REGION=$AWS_REGION
KUBERNETES_VERSION=$KUBERNETES_VERSION
RANCHER_VERSION=$RANCHER_VERSION
CERT_MANAGER_VERSION=$CERT_MANAGER_VERSION
DOMAIN_NAME=$DOMAIN_NAME
NLB_DNS_NAME=$NLB_DNS_NAME
RANCHER_TOKEN=$RANCHER_TOKEN
MASTER_PRIVATE_IP=$MASTER_PRIVATE_IP
CALICO_VERSION=$CALICO_VERSION
TIGERA_VERSION=$TIGERA_VERSION
ENABLE_BGP=$ENABLE_BGP
ENABLE_VXLAN=$ENABLE_VXLAN
ENABLE_WIREGUARD=$ENABLE_WIREGUARD
POD_CIDR=$POD_CIDR
SERVICE_CIDR=$SERVICE_CIDR
CLUSTER_DOMAIN=$CLUSTER_DOMAIN
OS=$OS
CREATED_AT=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
EOF

# ============================================================================== #
#  Verify installations
# ============================================================================== #
echo "[INFO] Verifying tool installations..."
kubectl version --client || true
helm version || true
k9s version || true