persistent-notif-perf.md

machine

 lsblk
NAME           MAJ:MIN RM   SIZE RO TYPE MOUNTPOINTS
sda              8:0    0 893.8G  0 disk
└─sda1           8:1    0 893.7G  0 part /
nvme1n1        259:0    0   1.5T  0 disk
nvme0n1        259:1    0   1.5T  0 disk
├─vg_nvme-lv_1 253:0    0  89.4G  0 lvm
├─vg_nvme-lv_2 253:1    0  89.4G  0 lvm
├─vg_nvme-lv_3 253:2    0  89.4G  0 lvm
├─vg_nvme-lv_4 253:3    0  89.4G  0 lvm
└─vg_nvme-lv_5 253:4    0  14.9G  0 lvm  /var/lib/ceph
nvme3n1        259:2    0   1.5T  0 disk
nvme4n1        259:3    0   1.5T  0 disk
nvme2n1        259:4    0   1.5T  0 disk
nvme6n1        259:5    0   1.5T  0 disk
nvme5n1        259:6    0   1.5T  0 disk
nvme7n1        259:7    0   1.5T  0 disk

setup the cluster

sudo MON=1 OSD=2 MDS=0 MGR=0 RGW=1 ../src/vstart.sh -n -X --nolockdep --bluestore \
    --bluestore-devs "/dev/nvme7n1,/dev/nvme6n1,/dev/nvme5n1,/dev/nvme4n1,/dev/nvme3n1,/dev/nvme2n1" \
    -o "bluestore_block_size=1500000000000" -o "rgw_max_concurrent_requests=8192" \
    -o "rgw_dynamic_resharding=false" -o "osd_pool_default_pg_num=128" -o "osd_pool_default_pgp_num=128" \
    -o "mon_max_pg_per_osd=32768" -o "mon_pg_warn_max_per_osd=32768" -o "osd_pool_default_pg_autoscale_mode=warn"

run hsbench baseline with 1M objects

hsbench -a 0555b35654ad1656d804 -s h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q== \
  -u http://localhost:8000 -bp bk -m ipd -t 64 -z 4K -l 1 -b 1

setup persistent notifications

aws --region=default --endpoint-url http://localhost:8000 sns create-topic --name=fishtopic0  \
  --attributes='{"push-endpoint": "kafka://localhost", "persistent": "true"}'
aws --region=default --endpoint-url http://localhost:8000 s3api put-bucket-notification-configuration \
  --bucket bk000000000000 --notification-configuration='{"TopicConfigurations": [{"Id": "notif1", "TopicArn": "arn:aws:sns:default::fishtopic0", "Events": []}]}'

run hsbench again

hsbench -a 0555b35654ad1656d804 -s h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q== \
  -u http://localhost:8000 -bp bk -m pd -t 64 -z 4K -l 1 -b 1

with the code that supports sharded queue. run baseline setting number of shards to 1:

sudo MON=1 OSD=2 MDS=0 MGR=0 RGW=1 ../src/vstart.sh -n -X --nolockdep --bluestore \
    -o "rgw_bucket_persistent_notif_num_shards=1" \
    --bluestore-devs "/dev/nvme7n1,/dev/nvme6n1,/dev/nvme5n1,/dev/nvme4n1,/dev/nvme3n1,/dev/nvme2n1" \
    -o "bluestore_block_size=1500000000000" -o "rgw_max_concurrent_requests=8192" \
    -o "rgw_dynamic_resharding=false" -o "osd_pool_default_pg_num=128" -o "osd_pool_default_pgp_num=128" \
    -o "mon_max_pg_per_osd=32768" -o "mon_pg_warn_max_per_osd=32768" -o "osd_pool_default_pg_autoscale_mode=warn"

compare results to a run with default number of shards

yuvalif/persistent-notif-perf.md