check handoffs vs. misplaced parts

classify_handoff_parts.py

#!/usr/bin/env python

import sys
import os
import errno
import json
from argparse import ArgumentParser
from collections import defaultdict
from itertools import islice

from swift.common.storage_policy import split_policy_string
from swift.obj.diskfile import get_data_dir
from swift.common.ring import Ring

parser = ArgumentParser()
group = parser.add_mutually_exclusive_group()
group.add_argument('-v', '--verbose', help='line oriented output',
                   default=False, action='store_true')
group.add_argument('-s', '--summary', help='json summary of all parts',
                   default=False, action='store_true')
parser.add_argument('ring', help='specify the ring, infers datadir')
parser.add_argument('devices', help='root of devices tree for node',
                    nargs='?', default='/srv/node')
parser.add_argument('--request-node-count',
                    help='max number of handoff device to check',
                    default=6, type=int)


def get_ring_and_datadir(path):
    """
    :param path: path to ring

    :returns: a tuple, (ring, datadir)
    """
    ring_name = os.path.basename(path).split('.')[0]
    base, policy = split_policy_string(ring_name)
    if base == 'object':
        datadir = get_data_dir(policy)
    else:
        datadir = base + 's'
    return Ring(path), ring_name, datadir


def main():
    args = parser.parse_args()
    device_root = args.devices

    ring, ring_name, datadir = get_ring_and_datadir(args.ring)

    dev2parts = defaultdict(set)
    for replica, part2dev in enumerate(ring._replica2part2dev_id):
        for part, device_id in enumerate(part2dev):
            dev2parts[ring.devs[device_id]['device']].add(part)

    dev2handoffs = defaultdict(set)
    searched = set()

    def is_handoff(device_dir, part):
        if part not in searched:
            if args.verbose:
                print 'get_more_nodes %s' % part
            for dev in islice(ring.get_more_nodes(part),
                              args.request_node_count):
                dev2handoffs[dev['device']].add(part)
            searched.add(part)
        return part in dev2handoffs[device_dir]

    # print dev2parts
    primary_count = defaultdict(int)
    handoffs = defaultdict(set)
    misplaced = defaultdict(set)
    device_dirs = os.listdir(device_root)
    for device_dir in device_dirs:
        parts_dir = os.path.join(device_root, device_dir, datadir)
        try:
            parts = os.listdir(parts_dir)
        except OSError as e:
            if e.errno == errno.ENOENT:
                continue
            else:
                raise
        for part in parts:
            if not part.isdigit():
                continue
            part = int(part)
            if part in dev2parts[device_dir]:
                primary_count[device_dir] += 1
                continue
            if is_handoff(device_dir, part):
                handoffs[device_dir].add(part)
            else:
                misplaced[device_dir].add(part)

    if args.summary:
        data = {ring_name: {"primary": sum(primary_count.values()),
                            "handoff": sum([len(y)
                                            for y in handoffs.values()]),
                            "misplaced": sum([len(z)
                                              for z in misplaced.values()])}}
        print json.dumps(data)
    else:
        print '%9s: %9s %9s %9s'.format('device',
                                        'primary',
                                        'handoff',
                                        'misplaced')
        for device in set(
                primary_count.keys() + handoffs.keys() + misplaced.keys()):
            print '%9s: %9d %9d %9d' % (
                device, primary_count[device],
                len(handoffs[device]),
                len(misplaced[device]),
            )
            if args.verbose:
                print ' handoffs: %s' % (','.join(
                    str(p) for p in handoffs[device]))
                print 'misplaced: %s' % (','.join(
                    str(p) for p in misplaced[device]))


if __name__ == "__main__":
    sys.exit(main())