check_file_hashes.py

import manage  # this initializes Django

import datetime
import hashlib
import pprint
import traceback
from collections import defaultdict

from amo.utils import chunked
from constants.base import VALID_STATUSES
from files.models import File


jan1_2015 = datetime.datetime(2015, 1, 1)
stats = defaultdict(int)
qs = (File.objects
      .filter(version__addon__status__in=VALID_STATUSES)
      .exclude(hash=''))
for chunk in chunked(qs, 300):
    for file_ in chunk:
        stats['checked'] += 1
        try:
            algo, db_hash = file_.hash.split(':')
            if algo != 'sha256':
                # Maybe this is really old file data or something.
                stats['skipped_algorithms'] += 1
                continue

            hasher = hashlib.sha256()
            with open(file_.current_file_path, 'rb') as f:
                for block in iter(lambda: f.read(1024), ''):
                    hasher.update(block)
            actual_hash = hasher.hexdigest()

            if db_hash != actual_hash:
                stats['hash_mismatch'] += 1
                if file_.is_signed:
                    stats['signed_hash_mismatch'] += 1
                if not file_.version.addon.is_listed:
                    stats['unlisted_hash_mismatch'] += 1
                if file_.created > jan1_2015:
                    stats['2015_hash_mismatch'] += 1
                print ('Wrong hash: {f.pk}:{f.filename}; '
                       'addon:{f.version.addon.pk}; actual:{actual}; '
                       'db:{db}; original:{original}'
                       .format(f=file_, actual=actual_hash,
                               db=db_hash, original=file_.original_hash))
            else:
                stats['hash_ok'] += 1

        except Exception, exc:
            print '{e.__class__.__name__}: {e}'.format(e=exc)
            stats['exceptions'] += 1
            continue

pprint.pprint(dict(stats))