find out source and destination cluster account - container conflict or account - container - object confliction

comparison.py

#!/usr/bin/python

import swiftclient
import pprint
import Queue
import threading
import time
import timeit
import keystoneclient
import ConfigParser
import sys

#global variant from config file
admin_user = ''
admin_pass = ''
source_auth_url = ''
source_swift_base = ''
destination_auth_url = ''
destination_swift_base = ''
n_tenant_threads = 1

#local variant use for code only
pp = pprint.PrettyPrinter(indent=4)
exitFlag = 0

def get_Config():
    global admin_user
    global admin_pass
    global source_auth_url
    global destination_auth_url
    global n_tenant_threads
    global account_list
    global source_swift_base
    global destination_swift_base
    global compare_object

    config = ConfigParser.ConfigParser()
    config.read(r'compare-clusters.conf')

    admin_user = config.get('global', 'admin_user')
    admin_pass = config.get('global', 'admin_pass')
    source_auth_url = config.get('global', 'source_auth_url')
    destination_auth_url = config.get('global', 'destination_auth_url')
    n_tenant_threads = config.getint('global', 'n_tenant_threads')
    account_list = config.get('global', 'account_list')
    source_swift_base = config.get('global', 'source_swift_base')
    destination_swift_base = config.get('global', 'destination_swift_base')
    compare_object = config.getboolean('global', 'compare_object')

class tenantThread(threading.Thread):
    def __init__(
        self,
        threadID,
        name,
        q,
        l,
        ):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.name = name
        self.q = q
        self.l = l
    def run(self):
        process_tenant_thread(self.name, self.q, self.l)

def process_tenant_thread(threadName, q, l):
    while not exitFlag:
        l.acquire()
        if not q.empty():
            data = q.get()
            l.release()
            #print '%s scanning Tenant %s' % (threadName, data)
            process_tenant(data)
        else:
            l.release()
        time.sleep(1)


def make_tenant_threadlist(int_tenant_threads):
    threadList = []
    for i in range(1, int_tenant_threads + 1):
        threadList.append('tenant Thread-' + str(i))

    return threadList

def process_tenant(data):
    print data
    global admin_user
    global admin_pass
    global source_auth_url
    global destination_auth_url
    global account_list
    global source_swift_base
    global destination_swift_base
    global compare_object

    try:
        # get source cluster connection by tenant
        swift_s = swiftclient.client.Connection(authurl=source_auth_url, user=admin_user, key=admin_pass, tenant_name=admin_user, auth_version='2.0',os_options={'object_storage_url': source_swift_base+data[1]})
        # get destination cluster connection by tenant
        swift_d = swiftclient.client.Connection(authurl=destination_auth_url, user=admin_user, key=admin_pass, tenant_name=admin_user, auth_version='2.0',os_options={'object_storage_url': destination_swift_base+data[1]})
        resp_headers, containers = swift_s.get_account()
        for c in containers:
            bolfoundcontainer = find_container(swift_d, c['name'])
            if bolfoundcontainer:
                print "source account: " + data[1] + " has container name: " + c['name'] + " found in destination: " + str(bolfoundcontainer)
            if compare_object and bolfoundcontainer:
                resp_headers, objects = swift_s.get_container(c['name'])
                for o in objects:
                    bolfoundobject = find_object(swift_d, c['name'], o['name'], o['hash'])
                    if bolfoundobject:
                        print "source account: " + data[1] + " has container name: " + c['name'] + " and object name: " + o['name'] + " & hash: " + o['hash'] + " found in destination: " + str(bolfoundobject)
        swift_s.close()
        swift_d.close()
    except Exception, err:
        print data[1] + ": " + str(err)

def find_container(swift_d, container_name):
    bolcheck = False
    try:
        resp_headers, containers = swift_d.get_account()
        for c in containers:
            if c['name'] == container_name:
                bolcheck = True
    except Exception:
        sys.exc_clear()
    return bolcheck

def find_object(swift_d, container_name, object_name, object_hash):
    bolcheck = False
    try:
        resp_headers, objects = swift_d.get_container(container_name)
        for o in objects:
            if ( o['name'] == object_name ) and ( o['hash'] == object_hash ):
                bolcheck = True
    except Exception:
        sys.exc_clear()
    return bolcheck

def get_tenant_list(account_list):
    with open(account_list) as f:
        lines = f.readlines()
    tenant_list = [x.strip() for x in lines]
    return tenant_list

def main():
    global n_tenant_threads
    global admin_user
    global admin_pass
    global source_auth_url
    global destination_auth_url
    global account_list

    start = timeit.default_timer()

    # Get Config
    get_Config()

    # Get Tenant list
    tenant_list = get_tenant_list(account_list)
    tenantThreadList = make_tenant_threadlist(n_tenant_threads)

    # Generate tenant dictionary by tennant list
    namedict = {}
    namecount = 1
    for tenant in tenant_list:
        namedict[namecount] = tenant
        namecount += 1

    # Queue # = thread list
    queueLock = threading.Lock()
    workQueue = Queue.Queue(0)  # queue size is infinite
    threads = []
    threadID = 1


    # Create new threads
    for tName in tenantThreadList:
        thread = tenantThread(threadID, tName, workQueue, queueLock)
        thread.start()
        threads.append(thread)
        threadID += 1

    # Fill the queue
    queueLock.acquire()

    # for word in nameList:
    for word in namedict.iteritems():
        workQueue.put(word)
    queueLock.release()

    # Wait for queue to empty
    while not workQueue.empty():
        pass

    # Notify threads it's time to exit
    global exitFlag
    exitFlag = 1

    # Wait for all threads to complete
    for t in threads:
        t.join()

    print 'Exiting Main Thread, Time Cost: %s' \
        % (timeit.default_timer() - start)

if __name__ == '__main__':
    try:
        main()
    except (KeyboardInterrupt):
        print "Abort, Got Keyboard Interrupt !"
        sys.exit(1)

configuration file example

$ cat compare-clusters.conf
[global]
admin_user=xxxxxxx
admin_pass=xxxxxxx
source_auth_url=https://source1.swiftstack.com/auth/v2.0
source_swift_base=https://source1.swiftstack.com/v1/
destination_auth_url=https://destination1.swiftstack.com/auth/v2.0
destination_swift_base=https://destination1.swiftstack.com/v1/
n_tenant_threads=10
account_list=test.csv
compare_object=True

account list example

$ cat test.csv
AUTH_ss

this account list came from https://github.com/swiftstack/ss-utilization with option --accountonly

output example

(ss-util) root@developer:~/test# python compare-clusters.py
(1, 'AUTH_ss')
 (3, 'AUTH_Fred_Hutchinson_Cancer_Research_Center-b6aa')
 (4, 'AUTH_HP-a8f5')
(2, 'AUTH_FactSet_Research-803c')
 (5, 'AUTH_PayPal-5b8e')
(6, 'AUTH_-0184')
 (7, 'AUTH_-0192')
 (9, 'AUTH_-026f')
(8, 'AUTH_-024a')
(10, 'AUTH_-0297')
source container name: .swiftstack-web-console-metadata found in destination: True
source object name: bookmarks2.json & hash: c877583c733d2333d3f0609cf3c4aa83 found in destination: True
source container name: test found in destination: True
source object name: 100MB & hash: 978fe39760e4d83e05c41579c35c3596 found in destination: True
Exiting Main Thread, Time Cost: 4.90257310867```