rexpire.py

"""
Re-expires objects based on previous expirer log output.
For example, to fix all the objects that expired during the time this bug
https://bugs.launchpad.net/swift/+bug/1257330 was deployed.
"""
CONF_PATH = '/some/path/rexpire.conf'  # Just a copy of object-expirer.conf
REQUEST_TRIES = 3
EXPIRING_OBJECTS_ACCOUNT = '.expiring_objects'
SERVICE_NAME = 'object-expirer'

import eventlet
import json
import sys
import time
import traceback
import urllib
import swift.common.internal_client

pool = eventlet.GreenPool(size=100)
client = swift.common.internal_client.InternalClient(CONF_PATH, 'Swift Object Expirer Clean Up', REQUEST_TRIES)

def check_on_object(account, container, obj, x_delete_at):
    try:
        resp = client.make_request('GET', '/v1/%s/%s?format=json&prefix=%s&limit=1' % (account, container, obj), {}, (2, 404))
        if resp.status_int == 404:
            return
        items = json.loads(resp.body)
        if not items:
            return
        item = items[0]
        if item['name'].encode('utf8') != urllib.unquote(obj):
            return
        size = item['bytes']
        resp = client.make_request('HEAD', '/v1/%s/%s/%s' % (account, container, obj), {}, (2, 404))
        if resp.status_int != 404:
            return
        # resp = client.make_request('DELETE', '/v1/%s/%s/%s' % (account, container, obj), {'X-If-Delete-At': str(x_delete_at)}, (2, 404, 412))
        sys.stdout.write('%s %s %s %s %d %d %d\n' % (time.asctime(time.gmtime()), account, container, obj, x_delete_at, time.time(), size))
    except Exception:
        sys.stderr.write('%s ERROR with %s %s %s %s:\n' % (time.asctime(time.gmtime()), account, container, obj, x_delete_at))
        traceback.print_exc()

for line in sys.stdin:
    try:
        if 'DELETE /v1/%s/' % EXPIRING_OBJECTS_ACCOUNT not in line:
            continue
        parts = line.split()
        if len(parts) < 10:
            continue
        if parts[4] not in (SERVICE_NAME, SERVICE_NAME + ':'):
            continue
        if parts[8] != 'DELETE':
            continue
        path_parts = parts[9].split('/', 6)
        if len(path_parts) != 7 or path_parts[0:3] != ['', 'v1', EXPIRING_OBJECTS_ACCOUNT]:
            continue
        if '-' not in path_parts[4]:
            continue
        x_delete_at, account = path_parts[4].split('-', 1)
        x_delete_at = int(x_delete_at)
        container = path_parts[5]
        obj = path_parts[6]
        pool.spawn_n(check_on_object, account, container, obj, x_delete_at)
        eventlet.sleep()
    except Exception:
        sys.stderr.write('%s ERROR with %s:\n' % (time.asctime(time.gmtime()), line))
        traceback.print_exc()
pool.waitall()