rgw_empty_explicit_placement.py

#!/usr/bin/env python

import sys,os,datetime
import simplejson as json
from subprocess import Popen, PIPE
from StringIO import StringIO

"""NOTE: Please ensure the 'CEPH_ARGS' environment variable is set for any cluster specific options required on radosgw-admin commands"""

cephargs=""
if 'CEPH_ARGS' in os.environ:
    cephargs=os.environ['CEPH_ARGS']

""" Process outline:
 1: Get list of all buckets
   # radosgw-admin bucket list
 2: Get 'stat' of bucket to identify bucket-id
   # radosgw-admin bucket stats --bucket={bucketname}

 if obj["explicit_placement"]["data_pool"]!=""
     3: Perform metadata get
       # radosgw-admin metadata get --metadata-key bucket.instance:{bucketname}:{bucketid} > {bucketname}.backup.instance.json
     4: Set all "explicit_placement" values to empty string "" and write remdiated.json file
     5: Remove existing metadata:
       # radosgw-admin metadata rm --rgw-cache-enabled=false --metadata-key bucket.instance:{bucketname}:{bucketid} --rgw-cache-enabled=false
     6: Perform metadata put
       # radosgw-admin metadata put --metadata-key bucket.instance:{bucketname}:{bucketid} < {bucketname}.instance.json
"""

def logLine(line):
    date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
    print "{0:s} {1:s}".format(date,line)

""" 
 1: Get list of all buckets
   # radosgw-admin bucket list
"""
logLine("Retrieving bucket list...")
bl_pipe = Popen(['radosgw-admin','bucket','list',cephargs],stdout=PIPE,stderr=PIPE)

bucketlist, myerr = bl_pipe.communicate()
if myerr!="":
    logLine("bucket listing error! :: {0:s}".format(myerr))
    sys.exit

bl_obj=json.load(StringIO(bucketlist))

for bucket in bl_obj:
    """
     2: Get 'stat' of bucket to identify bucket-id
       # radosgw-admin bucket stats --bucket={bucketname}
    """
    logLine("Getting bucket stats for {0:s}".format(bucket))
    bs_pipe = Popen(["radosgw-admin", "bucket","stats","--bucket={0:s}".format(bucket),cephargs],stdout=PIPE,stderr=PIPE)
    bucketstats, myerr = bs_pipe.communicate()
    if myerr!="":
        logLine("bucket stats error! :: {0:s}".format(myerr))
        continue
    bs_obj=json.load(StringIO(bucketstats))

    """
         if obj["explicit_placement"]["data_pool"]!=""
    """
    needRemediation=False
    for myfield in bs_obj["explicit_placement"].keys():
        if bs_obj["explicit_placement"][myfield]!="":
            needRemediation=True

    if needRemediation:
        """
         3: Perform metadata get
            # radosgw-admin metadata get --metadata-key bucket.instance:{bucketname}:{bucketid} > {bucketname}.backup.instance.json
        """
        logLine("Non-empty explicit placement data found for {0:s}. Getting metadata into {0:s}.backup.instance.json".format(bucket))
        retval=os.system("radosgw-admin metadata get --metadata-key bucket.instance:{0:s}:{1:s} {2:s} > {0:s}.backup.instance.json".format(bucket,bs_obj["id"],cephargs))

        if retval!=0:
            logLine("Failed to metadata get, exit code {0:d}".format(retval))
            continue

        """
         4: Set all "explicit_placement" values to empty string "" and write remdiated.json file
        """
        with open("{0:s}.backup.instance.json".format(bucket)) as md_file:
            md_obj=json.load(md_file)

        logLine("Setting empty strings for {0:s} explicit placement.".format(bucket))
        for myfield in md_obj["data"]["bucket_info"]["bucket"]["explicit_placement"].keys():
            md_obj["data"]["bucket_info"]["bucket"]["explicit_placement"][myfield]=""

        logLine("Writing for {0:s}.remediated.instance.json".format(bucket))
        with open("{0:s}.remediated.instance.json".format(bucket), 'w') as outfile:  
            json.dump(md_obj, outfile, indent=4,sort_keys=True,)

        """
         5: Remove existing metadata:
           # radosgw-admin metadata rm --rgw-cache-enabled=false --metadata-key bucket.instance:{bucketname}:{bucketid} --rgw-cache-enabled=false
        """
        logLine("Removing object metadata for {0:s}".format(bucket))
        retval=os.system("radosgw-admin metadata rm --rgw-cache-enabled=false --metadata-key bucket.instance:{0:s}:{1:s} --rgw-cache-enabled=false {2:s}".format(bucket,bs_obj["id"],cephargs))
        if retval!=0:
            logLine("Failed to metadata rm, exit code {0:d}".format(retval))
            continue

        """
         6: Perform metadata put
           # radosgw-admin metadata put --metadata-key bucket.instance:{bucketname}:{bucketid} < {bucketname}.instance.json
        """
        logLine("Performing metadata put for {0:s}.remediated.instance.json".format(bucket))
        retval=os.system("radosgw-admin metadata put --rgw-cache-enabled=false --metadata-key bucket.instance:{0:s}:{1:s} {2:s} < {0:s}.remediated.instance.json".format(bucket,bs_obj["id"],cephargs))
        if retval!=0:
            logLine("Failed to metadata put, exit code {0:d}".format(retval))
            continue

    else:
        logLine("Empty explicit placement for {0:s}. skipping".format(bucket))