Fetch list of Vim script as JSON

vimscript_list_json.py

#!/usr/bin/python3
# Fetch List of Vim Script as JSON


import io
import re
import urllib.request
import html.entities
import json


VIMSCRIPT_LIST_URL = "http://www.vim.org/scripts/script_search_results.php?&show_me=99999"


def htmlentityunescape(text):
    def fixup(m):
        text = m.group(0)
        if text[:2] == "&#":
            # character reference
            try:
                if text[:3] == "&#x":
                    return chr(int(text[3:-1], 16))
                else:
                    return chr(int(text[2:-1]))
            except ValueError:
                pass
        else:
            # named entity
            try:
                text = chr(html.entities.name2codepoint[text[1:-1]])
            except KeyError:
                pass
        return text  # leave as is
    return re.sub("&#?\w+;", fixup, text)


def vimscript_list():
    data = urllib.request.urlopen(VIMSCRIPT_LIST_URL).read().decode("ISO-8859-1")
    res = {}
    i = 0
    e = {}
    for line in io.StringIO(data):
        line = line.strip()
        m = re.search(r"rowodd|roweven", line)
        if m:
            if i == 0:
                e["script_id"] = int(re.search(r"script_id=(\d+)", line).group(1))
                e["name"] = htmlentityunescape(re.sub(r"<[^>]*>", "", line))
            elif i == 1:
                e["type"] = htmlentityunescape(re.sub(r"<[^>]*>", "", line))
            elif i == 2:
                e["rating"] = int(re.sub(r"<[^>]*>", "", line))
            elif i == 3:
                e["downloads"] = int(re.sub(r"<[^>]*>", "", line))
            elif i == 4:
                e["summary"] = htmlentityunescape(re.sub(r"<[^>]*>", "", line))
            if i == 4:
                res[e["script_id"]] = e
                i = 0
                e = {}
            else:
                i = i + 1
    return res


def main():
    print(json.dumps(vimscript_list()))


if __name__ == "__main__":
    main()