rados_reproduce.py

# /tmp/myobject is a 100k object
# created with:
#  dd if=/dev/urandom of=/tmp/myobject bs=100k count=1

import rados
import random
import socket

#objects name:  <oject-prefix>-<id>
object_prefix=socket.gethostname()
object_min_id=1
object_max_id=1800000
object_fs_path='/tmp/myobject'
fd=open(object_fs_path, 'r')
object_content=fd.read()
fd.close()

rados_pool='default.rgw.buckets.data'
cluster = rados.Rados(conffile='/etc/ceph/ceph.conf')
cluster.connect()
ioctx = cluster.open_ioctx(rados_pool)

def delete_random_obj():
    j = 0
    while j < 6:
        try:
            object_id=random.randint(object_min_id, object_max_id)
            object_name = object_prefix + '-' + str(object_id)
            ioctx.remove_object(object_name)
            print 'deleted obj: ' + object_name
            removed_object_ids.append(object_id)
            j += 1
        except:
            print 'error deleting obj. Likely DNE. Trying another one'
            j=j-1

def cleanup():
    print "-----------------------------------------------"
    print "|writing deleted objects. be patient           |"
    print "-----------------------------------------------"
    for id in removed_object_ids:
        object_name = object_prefix + '-' + str(id)
        print 're-writing: ' + object_name
        ioctx.write_full(object_name, object_content)
    print "-----------------------------------------------"
    print "|now deleting written objects. be patient      |"
    print "-----------------------------------------------"
    #delete newly written objects
    for id in range(object_max_id+1, write_object_id):
        try:
            object_name = object_prefix + '-' + str(id)
            ioctx.remove_object(object_name)
            print 'deleted: ' + object_name
        except:
            pass

#don't overwrite
write_object_id = object_max_id+1
i=1
removed_object_ids = []
try:
    while True:
        object_name = object_prefix + '-' + str(write_object_id)
        print 'writing obj: ' + object_name
        ioctx.write_full(object_name, object_content)
        i += 1
        write_object_id += 1

        if i > 10:
            delete_random_obj()
            i = 1
except KeyboardInterrupt:
    cleanup()