metatag_failure_count.py

import os
import warc
import math
from urlparse import urlparse

def get_failed_url_counts(links):
    """
    ## counting failed sites
    this can be done in perma prod, easiest way to get to link model
    """
    from perma.models import Link
    failed_sites = dict()
    for link in links:
        url_base = urlparse(link.submitted_url).netloc
        if url_base in failed_sites:
            failed_sites[url_base] += 1
        else:
            failed_sites[url_base] = 1
    sorted_failed = sorted(failed_sites, key=failed_sites.get, reverse=True)
    with open('failed_sites.txt', 'w+') as f:
        for failed in sorted_failed:
            f.write("%s: %s \n" % (failed_sites[failed], failed))

def get_failed_resource_counts(failed_guids_file, failed_warc_sizes_file, failed_resource_links_file, failed_resources_fullpath_file):
    """
    ## counting resources of failed archives
    this has to be done locally because we need a few packages that don't exist in perma prod
    - download all warcs from perma-mirror
    - $ rsync -avz --files-from=failed_guids_file perma-mirror:/ your-results-dir/.

    """
    failed_resources = dict()
    failed_resources_fullpath = dict()
    with open(failed_guids_file, "rb+") as f:
        for l in f.readlines():
            filepath = l.split('\n')[0]
            # make sure that the filepath reflects your current directory here
            # filepath = "your-results-dir" + filepath
            try:
                file_size = os.path.getsize(filepath)
                guid = filepath.split('/')[-1].split('.warc.gz')[0]
                with open(failed_warc_sizes_file, 'a+') as f:
                    f.write("%s %s\n\n" % (guid, convert_size(file_size)))

                warc_open = warc.open(filepath)
                for record in warc_open:
                    if record.type == 'response':
                        fullpath = record.url
                        parts = urlparse(fullpath)
                        path = "%s://%s%s" % (parts.scheme, parts.netloc, parts.path)
                        if path in failed_resources:
                            failed_resources[path] += 1
                        else:
                            failed_resources[path] = 1

                        if fullpath in failed_resources_fullpath:
                            failed_resources_fullpath[fullpath] += 1
                        else:
                            failed_resources_fullpath[fullpath] = 1
            except:
                pass

    sorted_failed_resource_links = sorted(failed_resources, key=failed_resources.get, reverse=True)
    sorted_failed_fullpath = sorted(failed_resources_fullpath, key=failed_resources_fullpath.get, reverse=True)

    with open(failed_resource_links_file, 'w+') as f:
        for failed in sorted_failed_resource_links:
            f.write("%s: %s \n" % (failed_resources[failed], failed))

    with open(failed_resources_fullpath_file, 'w+') as f:
        for failed in sorted_failed_fullpath:
            f.write("%s: %s \n" % (failed_resources_fullpath[failed], failed))

def convert_size(size_bytes):
   """taken from https://stackoverflow.com/questions/5194057/better-way-to-convert-file-sizes-in-python"""
   if (size_bytes == 0):
       return '0B'
   size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
   i = int(math.floor(math.log(size_bytes, 1024)))
   p = math.pow(1024, i)
   s = round(size_bytes/p, 2)
   return '%s %s' % (s, size_name[i])

failed_links = list(Link.objects.filter(tags__name__in=['meta-tag-retrieval-failure']))
get_failed_url_counts(failed_links)
get_failed_resource_counts(failed_guids_file, failed_warc_sizes_file, failed_resource_links_file, failed_resources_fullpath_file)