Control Plane Migration OCP-Virt

STEPS.MD

Steps

1. Identify ETCD Master

etcdctl endpoint status -w table

2. Shutdown non leader node and wait for node to report "Not Ready" and API Server to stabilize
3. Update Quorum Guard and wait for API Server to stabilize (control plane components should redeploy)

oc patch etcd/cluster --type=merge \
  -p '{"spec": {"unsupportedConfigOverrides": {"useUnsupportedUnsafeNonHANonProductionUnstableEtcd": true}}}'

4. Remove Member from ETCD

etcdctl member list
etcdctl member remove ${MEMBER_ID}

5. Remove Certificate Secrets for Node

oc delete secret -n openshift-etcd etcd-peer-${HOSTNAME}
oc delete secret -n openshift-etcd etcd-serving-${HOSTNAME}
oc delete secret -n openshift-etcd etcd-serving-metrids-${HOSTNAME}

6. Delete Node

oc delete node ${HOSTNAME}

7. Create Network Connection File and Butane Configuration File

variant: openshift
version: 4.19.0
metadata:
  name: example
  labels:
    machineconfiguration.openshift.io/role: master
storage:
  files:
  - path: /etc/sysconfig/network-scripts/ifcfg-${INTERFACE_NAME}
    contents:
      inline: |
        NAME="${INTERFACE_NAME}"
        DEVICE="${INTERFACE_NAME}"
        ONBOOT=yes
        NETBOOT=yes
        BOOTPROTO=none
        IPADDR="${STATIC_IP}"
        NETMASK="${NETMASK}"
        GATEWAY="${GATEWAY_IP}"
        TYPE=Ethernet
        DNS1="${DNS_IP}"

8. Create Ignition from Butane and UserData

butane --pretty --strict ${BUTANE_FILE} | yq -o json | jq -r tostring

9. Obtain Master Ignition from Kubernetes Secret (on Virtualized Cluster)

oc get secrets -n openshift-machine-api master-user-data -o yaml

10. Upload Kubernetes Secret with Master Ignition and Custom Ignition

apiVersion: v1
kind: Secret
metadata:
  name: ignition-payload
  namespace: default
type: Opaque
stringData:
  userdata: |
    {
      "ignition": {
        "config": {
          "merge": [
            {
              "source": "https://172.16.22.100:22623/config/master",
              "verification": {}
            }
          ],
          "replace": {
            "verification": {}
          }
        },
        "proxy": {},
        "security": {
          "tls": {
            "certificateAuthorities": [
              {
                "source": "data:text/plain;charset=utf-8;base64,LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURFRENDQWZpZ0F3SUJBZ0lJWko5UXdmcmVVWmN3RFFZSktvWklodmNOQVFFTEJRQXdKakVTTUJBR0ExVUUKQ3hNSmIzQmxibk5vYVdaME1SQXdEZ1lEVlFRREV3ZHliMjkwTFdOaE1CNFhEVEkxTURZeU5ERTFNamt4TWxvWApEVE0xTURZeU1qRTFNamt4TWxvd0pqRVNNQkFHQTFVRUN4TUpiM0JsYm5Ob2FXWjBNUkF3RGdZRFZRUURFd2R5CmIyOTBMV05oTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUEycjFKUnNvMVBMS1EKTlVuUFBiaEdFYm13OVJQZHI3Qkw5TE9Da2t5eDNOV1NiWEpVM042R3NYa2RQUi9XOFZxMVFVTnA4RzdYeXgyawpiQW43MWZyUEErR28wb2R5U252Z2p6VnM4L25VRG5sb1p2dUxKWU1QRG9OR2dMVDB2RFl1ckVaMzFXdHpnVk1yClNQUEw2ekdEL3BOTlV5TnNVcXBNekJ2dm15OGVGV3kzaG1maWVudW9oSFFtb2oxRVF0RStwOGtQMnRvb1VXYlUKN2lIM1RDNyt3T3pZVW1tMzJWM1RsN1FvMWdYNWM5eUNqUUdmUU1Kbk1PNmQ1eVd1YjhxMUJ2Z0N0U1dXN1huQQpDU09MLzlTVTNWSkt1aTZxYWdmaFRFeWhsck96eDFCelpZeHNvQmlrTDIzQlZvNEMyYjNZWmxIcUhzbCtxUVRICkMrQ2w5VEVjS3dJREFRQUJvMEl3UURBT0JnTlZIUThCQWY4RUJBTUNBcVF3RHdZRFZSMFRBUUgvQkFVd0F3RUIKL3pBZEJnTlZIUTRFRmdRVUtNK1ZVeEpDcTZGcys0eDRJNUJYU0ZtNVpsVXdEUVlKS29aSWh2Y05BUUVMQlFBRApnZ0VCQUxJNVRXSm80cHM0RDY1ZC8zMC9aUnI1NmRrT2RyR09JU0sxaFkxUmhwVEdpMDEwU0VKbHdnNkpzTHY0Cmp4Q2tvalJTSmZhcDNxcHJUVUoxMlNJR3JINm9KNXptVFpRSlErRk1UTDBrRFRvTUhnWE93SFEvUjI3TjA0LysKaW9KN3pvTk80cjZkd3c3YkZNK0RUMzJKcGt6N2dqQ1B1dGovUFRoRVlZTHlPbGtyRUpwUGc1QW4vTytHN3RlQwpvc0o3eTJ4R0xGL2oyVlhKQStqWmJKTFM0RG1rSy9WQ1poWTdiQjJZNkZMbTFhNm85bGE0UnlpR3JHVXRPYXNVCkZKODZKTHB3Z1lEbVZNYnlvVUcvdnpjNWdRQ1l5bGxyckl1QXJGcVNEeVVvMnZBeXdXTytPWmpLMldZQkNhd1cKeDZRRmU1QWQraGhBSDFJcnRjWGNvVC9BRjZnPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==",
                "verification": {}
              }
            ]
          }
        },
        "timeouts": {},
        "version": "3.5.0"
      },
      "kernelArguments": {},
      "passwd": {},
      "storage": {
        "files": [
          {
            "contents": {
              "compression": "gzip",
              "source": "data:;base64,H4sIAAAAAAAC/1zKwQrCMAzG8XseIw9Q2uL0lEO1QYasLVtQ9gABT1XcLr691MMQIYH/B78UBibU+nSLRYh87U8/O6djzkJvXSCxbN2ijFky1UdV6EuIcSR0B2/c3nhvPDY/hOlC6LvOfH9njUU4B+FbmDfdDkHmwsTrXV9VV4hpcn/gEwAA///XESE+qAAAAA=="
            },
            "mode": 384,
            "path": "/etc/sysconfig/network-scripts/ifcfg-enp1s0"
          }
        ]
      },
      "systemd": {}
    }    

11. Acquire Appropriate OpenShift Image

oc image extract --file /release-manifests/0000_50_installer_coreos-bootimages.yaml ${RELEASE_VERSION_IMAGE} --confirm
cat 0000_50_installer_coreos-bootimages.yaml | yq -r .data.stream | jq -r '.architectures.x86_64.images.kubevirt."digest-ref"'

12. Create Registry Secret (if necessary)

kind: Secret
apiVersion: v1
metadata:
  name: registry-secret
  namespace: default
stringData:
  accessKeyId: ${USERNAME}
  secretKey: ${PASSWORD}
type: Opaque

13. Remove Node from Kubernetes

oc delete node ${NODE}

14. Create new master node with Kubernetes Secret and OpenShift Image (It will install to disk and reboot)

apiVersion: kubevirt.io/v1
kind: VirtualMachine
metadata:
  name: ${HOSTNAME}
spec:
  runStrategy: Always
  dataVolumeTemplates:
    - metadata:
        name: dv-rhcos-${HOSTNAME}
      spec:
        pvc:
          accessModes:
          - ReadWriteOnce
          resources:
            requests:
              storage: 200Gi
          storageClassName: ${STORAGE_CLASS_NAME}
        source:
          registry:
            url: docker://${COS_IMAGE_NAME}
            secretRef: registry-secret
  template:
    spec:
      domain:
        # Needs to meet OCP minimum CPU/RAM
        cpu:
          cores: 1
          sockets: 8
          threads: 1
        memory:
          guest: 16Gi
        devices:
          disks:
            - disk:
                bus: virtio
              name: disk-rhcos-${HOSTNAME}
            - name: cloudinitdisk
              disk:
                bus: virtio
          interfaces:
            - model: virtio
              name: ${HOSTNAME}
              bridge: {}
      networks:
        - name: ${HOSTNAME}
          multus:
            networkName: ${NET_ATTACH_DEF_NS}/${NET_ATTACH_DEF_NAME}
      volumes:
        - name: cloudinitdisk
          cloudInitConfigDrive:
            secretRef:
              name: ignition-payload
        - dataVolume:
            name: dv-rhcos-${HOSTNAME}
          name: disk-rhcos-${HOSTNAME}

15. Approve Node CSRs (Two Rounds)

oc get csr -o go-template='{{range .items}}{{if not .status}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' | xargs --no-run-if-empty oc adm certificate approve
oc get csr -o go-template='{{range .items}}{{if not .status}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' | xargs --no-run-if-empty oc adm certificate approve

16. Repeat Steps for additional nodes
17. Reset Quorum Guard

oc patch etcd/cluster --type json -p '[{"op":"remove", "path": "/spec/unsupportedConfigOverrides"}]'

18. Delete Original Virtual Machine