Migrate EBS Volume based PVs across AWS availability zones (ref. https://gist.github.com/caruccio/be825aa39d53535217494369cc793dbd)

migrate-pv-to-zone.sh

#!/bin/bash

set -eu

NAMESPACE=$1
PVCNAME=$2
TARGETZONE=$3

# SNAPSHOTID is optional, if you did a snapshot already, you can pass it
SNAPSHOTID=${4:-}

PVNAME=$(kubectl -n $NAMESPACE get pvc $PVCNAME --template={{.spec.volumeName}})

VOLUMEID=$(kubectl -n $NAMESPACE get pv $PVNAME --template={{.spec.awsElasticBlockStore.volumeID}} | cut -d/ -f 4)
if [[ "$VOLUMEID" == "<no value>" ]]; then
  VOLUMEID=$(kubectl -n $NAMESPACE get pv $PVNAME --template={{.spec.csi.volumeHandle}})
fi

# DEPLOYMENTOBJ=$5
# REPLICAS=$(kubectl -n $NAMESPACE get $DEPLOYMENTOBJ --template={{.spec.replicas}})
# echo Deployment object -- $DEPLOYMENTOBJ
# echo Original Replicas -- $REPLICAS

# e.g. eu-west-1a -> eu-west-1
REGION=$(echo $TARGETZONE | sed -nr 's|(.*)(-[1-9])[a-z]|\1\2|p')

cat <<EOF
Summary
 Namespace ---------- $NAMESPACE
 PVC Name ----------- $PVCNAME
 PV Name ------------ $PVNAME
 Target Zone -------- $TARGETZONE
 REGION -------- $REGION
 Volume Id ---------- $VOLUMEID
EOF

read -p 'Press ENTER to continue'
echo

# if [ $REPLICAS -gt 0 ]; then
#     echo -n "Scaling down $DEPLOYMENTOBJ: $REPLICAS -> 0"
#     kubectl -n $NAMESPACE scale --replicas=0 $DEPLOYMENTOBJ
#     while [ "$(kubectl -n $NAMESPACE get $DEPLOYMENTOBJ --template={{.status.replicas}})" != "0" ]; do
#         echo
#     done
# 
#     while sleep 0.1; do
#         echo
#         kubectl -n $NAMESPACE get pod
#         read -p "Press ENTER to continue" -t 3 || continue
#         break
#     done
# 
# fi

DESCRIPTION="snapshot-migration-${NAMESPACE}_${PVCNAME}_${VOLUMEID}"

echo -n "Creating snapshot... "
if [[ -z $SNAPSHOTID ]]; then
  SNAPSHOTID=$(aws ec2  create-snapshot  --volume-id $VOLUMEID --description $DESCRIPTION --output text --query SnapshotId)
fi
echo "Waiting for Snapshot ID: $SNAPSHOTID"

SNAPSHOTPROGRESS=$(aws ec2 describe-snapshots --snapshot-ids $SNAPSHOTID --query "Snapshots[*].Progress" --output text)
while [ $SNAPSHOTPROGRESS != "100%" ]
do
  sleep 15
  echo "Snapshot ID: $SNAPSHOTID $SNAPSHOTPROGRESS"
  SNAPSHOTPROGRESS=$(aws ec2 describe-snapshots --snapshot-ids $SNAPSHOTID --query "Snapshots[*].Progress" --output text)
done
# aws ec2 wait snapshot-completed --filter Name=snapshot-id,Values=$SNAPSHOTID
aws ec2 wait snapshot-completed --snapshot-ids "$SNAPSHOTID"


echo -n "Creating volume... "
TAGSPEC="ResourceType=volume,Tags=[{Key=Name,Value=kubernetes-dynamic-$PVNAME},{Key=kubernetes.io/created-for/pv/name,Value=$PVNAME},{Key=kubernetes.io/created-for/pvc/name,Value=$PVCNAME},{Key=kubernetes.io/created-for/pvc/namespace,Value=$NAMESPACE}]"
VOLUMEID=$(aws ec2 create-volume \
    --availability-zone $TARGETZONE \
    --snapshot-id $SNAPSHOTID \
    --volume-type gp3 \
    --output text \
    --query VolumeId \
    --tag-specifications "$TAGSPEC")
echo $VOLUMEID

# NOTE: can't update the pv directly. It will give you "Forbidden: is immutable after creation" error.
# echo Updating $DEPLOYMENTOBJ...
# kubectl label pv/$PVNAME failure-domain.beta.kubernetes.io/zone=$TARGETZONE --overwrite
# kubectl patch -p "{\"spec\":{\"awsElasticBlockStore\":{\"volumeID\":\"aws://$TARGETZONE/$VOLUMEID\"}}}" pv/$PVNAME

kubectl get pv/$PVNAME -ojson > $PVCNAME.old.$PVNAME.json
cat $PVCNAME.old.$PVNAME.json | jq 'del(.status)' | sed -r "s|$REGION[a-z]|$TARGETZONE|" | sed -r "s|vol-[a-z0-9]+|$VOLUMEID|" > $PVCNAME.new.$PVNAME.json

echo Please check the generated $PVCNAME.new.$PVNAME.json
$EDITOR $PVCNAME.new.$PVNAME.json

echo "If it is all good, going to delete the old PV and apply the new PV"
read -p 'Press ENTER to continue'
echo

kubectl delete pv $PVNAME &
delete_pid=$!
kubectl patch pv/$PVNAME -p '{"metadata":{"finalizers":null}}'

wait $delete_pid

kubectl apply -f $PVCNAME.new.$PVNAME.json

echo Wait the status of the PVC become Bound from Lost
kubectl -n $NAMESPACE get pvc $PVCNAME -w

# if [ $REPLICAS -gt 0 ]; then
#     echo "Scaling back $DEPLOYMENTOBJ: 0 -> $REPLICAS"
#     kubectl -n $NAMESPACE scale --replicas=$REPLICAS $DEPLOYMENTOBJ
# fi

# Finally, you need to manually delete the old volume and snapshot