mirroring-bz-161.sh

#!/bin/bash
set -ex

echo "===setup clusters ==="
mon=1 osd=1 mds=0 ../src/mstart.sh site-a --short -n --without-dashboard
mon=1 osd=1 mds=0 ../src/mstart.sh site-b --short -n --without-dashboard

echo "===rbd mirror setup for clusters ==="
key=$(../src/mrun site-a ceph auth get client.admin --format=json | jq -r .[0].key)
mon_host_a=$(grep "mon host" run/site-a/ceph.conf | awk '{print $4}')
ceph_conf=./run/site-a/ceph.conf bin/rbd-mirror --mon_host $mon_host_a --key $key --debug-rbd 5 --debug-rbd-mirror 30
mon_host_b=$(grep "mon host" run/site-b/ceph.conf | awk '{print $4}')
key=$(../src/mrun site-b ceph auth get client.admin --format=json | jq -r .[0].key)
ceph_conf=./run/site-b/ceph.conf bin/rbd-mirror --mon_host $mon_host_b --key $key --debug-rbd 5 --debug-rbd-mirror 30

echo "===create pools=="
../src/mrun site-a ceph osd pool create data
../src/mrun site-b ceph osd pool create data

echo "make them rbd pool...\n"
../src/mrun site-a rbd pool init data
../src/mrun site-b rbd pool init data

echo "===connect the 2 site clusters=="
../src/mrun site-a rbd mirror pool enable --site-name site-a data image
token=$(../src/mrun site-a rbd mirror pool peer bootstrap create data)
../src/mrun site-b rbd mirror pool peer bootstrap import --site-name site-b data - <<< $token

echo "===enable snapshot on ext4 rbd device for site-a==="
../src/mrun site-a rbd create --size 1G --mirror-image-mode snapshot --image-feature layering data/a1
dev=$(sudo bin/rbd device map --conf ./run/site-a/ceph.conf --device-type nbd data/a1)
sudo mkfs.ext4 $dev
sudo bin/rbd device unmap --conf ./run/site-a/ceph.conf --device-type nbd $dev

echo "enable snapshot mode..."
../src/mrun site-a rbd mirror image enable data/a1 snapshot

echo "add snap schedule..."
../src/mrun site-a rbd mirror snapshot schedule add --pool data --image a1 2m

echo "schedule ls"
../src/mrun site-a rbd mirror snapshot schedule status

echo "snap ls"
../src/mrun site-a rbd snap ls data/a1 --all

echo "demote site-a"
../src/mrun site-a rbd mirror image demote data/a1

echo "check demotion of site-a ..."
../src/mrun site-a rbd snap ls data/a1 --all

echo "promote site-b"
../src/mrun site-b rbd mirror image promote data/a1

echo "add schedule to site-b"
../src/mrun site-b rbd mirror snapshot schedule add --pool data --image a1 2m

echo "schedule status check"
../src/mrun site-b rbd mirror snapshot schedule ls --recursive

echo "demote site-b"
../src/mrun site-b  rbd mirror image demote data/a1

echo "promote site-a again"
../src/mrun site-a rbd mirror image promote data/a1

echo "snap list from site-a"
../src/mrun site-a rbd snap ls data/a1 --all

echo "snap status check for site-a"
../src/mrun site-a rbd mirror snapshot schedule status

echo "try failove to site-b"
../src/mrun site-a rbd mirror demote data/a1
../src/mrun site-b rbd mirror promote data/a1

echo "schedule and snap status for site-b"
../src/mrun site-b rbd mirror snapshot schedule status
../src/mrun site-b rbd mirror snap ls data/a1 --all
sleep 125

echo "check if snaps taken or not"
../src/mrun site-b rbd mirror snap ls data/a1 --all

echo "add schedule again on site-b"
../src/mrun site-b rbd mirror snapshot schedule add --pool data --image a1 2m

echo "wait 2mins..."
sleep 125
../src/mrun site-b rbd mirror snapshot schedule status

the local image has following status:

[ideepika@vossi03 build]$ ../src/mrun site-b rbd mirror image status data/a1
2021-11-17T12:21:06.981+0000 7fa4f7317200 -1 WARNING: all dangerous and experimental features are enabled.
2021-11-17T12:21:06.982+0000 7fa4f7317200 -1 WARNING: all dangerous and experimental features are enabled.
2021-11-17T12:21:06.985+0000 7fa4f7317200 -1 WARNING: all dangerous and experimental features are enabled.
a1:
  global_id:   83bdfbaa-c594-4845-8cdc-84924298ec3d
  state:       up+stopped
  description: local image is primary
  service:     admin on vossi03
  last_update: 2021-11-17 12:21:03
  peer_sites:
    name: site-a
    state: up+replaying
    description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"local_snapshot_timestamp":1637149530,"remote_snapshot_timestamp":1637149530,"replay_state":"idle"}
    last_update: 2021-11-17 12:21:01
  snapshots:
    14 .mirror.primary.83bdfbaa-c594-4845-8cdc-84924298ec3d.d577b041-ddd6-47e1-8fcc-1373e34301bd (peer_uuids:[a661e1dc-50e1-4cc8-9a7c-e981f6718bab])