Mirror a conda repository

check_repodata.py

#!/usr/bin/env python
from __future__ import print_function
import json
import os
import sys

def check(path, checksum=False):
    if checksum:
        from conda.utils import md5_file
    missing = set()
    bad = set()
  
    with open(os.path.join(path, 'repodata.json')) as f:
        repodata = json.load(f)
    
    for fname, info in repodata['packages'].iteritems():
        fpath = os.path.join(path, fname)
        try:
            size = os.path.getsize(fpath)
        except OSError:
            missing.add(fname)
        else:
            if size != int(info['size']):
                bad.add(fname)
            elif checksum:
                md5 = md5_file(fpath)
                if md5 != info['md5']:
                    bad.add(fname)
    
    return missing, bad

def main():
    import argparse
    parser = argparse.ArgumentParser()
    g = parser.add_mutually_exclusive_group()
    g.add_argument('--checksum', action='store_true', default=False)
    g.add_argument('--no-checksum', action='store_false', dest='checksum')
    g = parser.add_mutually_exclusive_group()
    g.add_argument('--delete-bad', action='store_true', default=False)
    g.add_argument('--no-delete-bad', action='store_false', dest='delete_bad')
    parser.add_argument('dir')
    args = parser.parse_args()
    
    missing, bad = check(args.dir, checksum=args.checksum)
    
    s = ", deleting" if args.delete_bad else ""
    if bad:
        print("These files are bad{}:".format(s))
        for fn in bad:
            print("\t{}".format(fn))
            if args.delete_bad:
                os.remove(os.path.join(args.dir, fn))
    
    if missing:
        print("Missing these packages:")
        for fname in sorted(missing):
            print("\t{}".format(fname))
    
    sys.exit(bool(missing) or bool(bad))


if __name__ == '__main__':
    main()

mirror.sh

#!/bin/bash

url="$(echo $1 | sed -E 's|/$||')"
path="${2:-$(echo $url | sed -E 's|^https?://||')}"

cut_dirs=$(echo "$url" | sed -E 's|^https?://()/?$|\1/|' | tr -Cd / | wc -c)

wget -N -P "$path" "$url"/repodata.json{,.bz2}
wget -r -nc -nH -P "$path" --cut-dirs $cut_dirs "$url/"
$(dirname $0)/check_repodata.py "$path"