get_underived_thumper_items.py

#!/usr/bin/env python
import datetime
import logging
import sys

import ujson

# parallel_md_get.py available here: https://gist.github.com/3784845
from parallel_md_get import metadata_record_iterator


date = datetime.datetime.utcnow().strftime("%Y-%m-%d")
log_filename = "%s-%s.log" % (__file__.strip('./py'), date)
logging_format = "%(asctime)s\t%(levelname)s\t%(message)s"
logging.basicConfig(filename=log_filename,level=logging.WARNING,
                    format=logging_format)


def item_is_derived(formats):

    arc_formats = [
            'Internet Archive ARC GZ',
            'Encrypted Internet Archive ARC GZ',
            'Web ARChive GZ',
    ]

    cdx_formats = [
            'ARC CDX Index',
            'Item CDX Index',
            'Item CDX Meta-Index',
            'WARC CDX Index',
    ]

    if any(f in formats for f in arc_formats) is True:
        if any(f in formats for f in cdx_formats) is False:
            return False
    return True

#_______________________________________________________________________________
ids = open(sys.argv[1])
results = metadata_record_iterator(ids, workers=20)

for i, id, md_json in results:
    try:
        metadata = ujson.loads(md_json)
        files = metadata.get('files')

        if not files:
            logging.warning("item has no files!\t%s" % id)
            continue

        formats = set(f['format'] for f in files)
        if item_is_derived(formats) is False:
            output_file = 'underived-thumper-items_%s.txt' % date
            with open(output_file, 'a') as f:
                f.write('%s\n' % id)

    except Exception, error:
        logging.error("%s:\t%s" % (id, error))