arch-void-compare.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import re
import requests
import urllib2
from distutils.version import LooseVersion

def urldecode(s):
    return urllib2.unquote(s).decode('utf8')

def unique_nested_list(nested_list):
    return [list(y) for y in set([tuple(x) for x in nested_list])]

def build_package_str(package):
    return "%s | Void: %s | Arch: %s\n" % (package["name"], package["void"], package["arch"])

regex_arch = r"href=\"(.+?)-([\d\.r]+)-.+\">"
regex_void = r"href=\"(.+)-([\d\.r]+)_.+\">"

urls_arch = [
    "http://mirrors.kernel.org/archlinux/core/os/x86_64/",
    "http://mirrors.kernel.org/archlinux/community/os/x86_64/",
    "http://mirrors.kernel.org/archlinux/extra/os/x86_64/"
]

urls_void = [
    "https://repo.voidlinux.eu/current/"
]


html_arch = ""
html_void = ""

print "Fetching arch packages"
for url in urls_arch:
    print url
    html_arch += requests.get(url).text

print "\nFetching void packages"
for url in urls_void:
    print url
    html_void += requests.get(url).text

matches_arch = re.findall(regex_arch, html_arch)
matches_void = re.findall(regex_void, html_void)

# url decode & lowercase packages
packages_arch = [[urllib2.unquote(item).decode('utf8').lower() for item in match] for match in matches_arch]
packages_void = [[urllib2.unquote(item).decode('utf8').lower() for item in match] for match in matches_void]

# unique and convert to dictionary
packages_arch = { package[0]: package[1:][0] for package in unique_nested_list(packages_arch) }
packages_void = { package[0]: package[1:][0] for package in unique_nested_list(packages_void) }

print "\nFound: %s arch packages" % len(packages_arch)
print "Found: %s void packages" % len(packages_void)

print "\nComparing versions"

up_to_date = []
outdated = []
newer = []
not_found = []

# compare versions
for package_name, package_version in packages_void.iteritems():
    # package found in arch repo
    if package_name in packages_arch:
        package = {
            "name": package_name,
            "arch": packages_arch[package_name],
            "void": packages_void[package_name]
        }

        if LooseVersion(package_version) == LooseVersion(packages_arch[package_name]):
            up_to_date.append(package)

        elif LooseVersion(package_version) < LooseVersion(packages_arch[package_name]):
            outdated.append(package)

        elif LooseVersion(package_version) > LooseVersion(packages_arch[package_name]):
            newer.append(package)

    # package not found in arch repo
    else:
        package = {
            "name": package_name,
            "arch": "not found",
            "void": packages_void[package_name]
        }
        not_found.append(package)

with open("output.txt", "w") as f:
    f.write("up to date (%s packages)\n-------------------------\n" % len(up_to_date))
    for package in up_to_date:
        f.write(build_package_str(package))

    f.write("\noutdated (%s packages)\n-------------------------\n" % len(outdated))
    for package in outdated:
        f.write(build_package_str(package))

    f.write("\nnewer (%s packages)\n-------------------------\n" % len(newer))
    for package in newer:
        f.write(build_package_str(package))

    f.write("\nnot found (%s packages)\n-------------------------\n" % len(not_found))
    for package in not_found:
        f.write(build_package_str(package))

print "\nDone. Wrote output.txt. Exiting"