AMO locales data

amo_localization_data.py

#!/usr/bin/env python3

import json
import re
import sys
from urllib.parse import quote as urlquote
from urllib.request import urlopen


def main():
    # Get list of frontend locales (langs array)
    # https://github.com/mozilla/addons-frontend/blob/master/config/default.js#L182-L249
    try:
        print("Reading frontend locales...")
        frontend_url = "https://raw.githubusercontent.com/mozilla/addons-frontend/master/config/default.js"
        response = urlopen(frontend_url)
        js_content = response.read().decode("utf-8")
        match = re.search(r"langs\s*:\s*\[(.*?)\]", js_content, re.DOTALL)

        if match:
            js_array = match.group(1)  # Extract the contents inside the array brackets
            frontend_locales = [
                loc.replace('"', "").replace("'", "").strip()
                for loc in js_array.split(",")
            ]
            frontend_locales = [loc for loc in frontend_locales if loc]
            frontend_locales.sort()
    except Exception as e:
        print(e)

    # Get list of server locales (AMO_LANGUAGES dictionary keys)
    # https://github.com/mozilla/addons-server/blob/master/src/olympia/core/languages.py#L3-L104
    try:
        print("Reading server locales...")
        server_url = "https://raw.githubusercontent.com/mozilla/addons-server/master/src/olympia/core/languages.py"
        response = urlopen(server_url)
        content = response.read().decode("utf-8")
        local_namespace = {}
        exec(content, {}, local_namespace)

        server_locales = list(local_namespace["AMO_LANGUAGES"].keys())
        server_locales.sort()

    except Exception as e:
        print(e)

    projects = [
        "amo",
        "amo-frontend",
    ]

    # Get stats from Pontoon
    query = """
{
    projects {
        name
        slug
        localizations {
            locale {
                code
            },
            approvedStrings,
            stringsWithWarnings,
            missingStrings,
            pretranslatedStrings,
            totalStrings
        }
    }
}
"""
    locale_data = {}
    try:
        print("Reading Pontoon stats...")
        url = f"https://pontoon.mozilla.org/graphql?query={urlquote(query)}"
        response = urlopen(url)
        json_data = json.load(response)

        for project in json_data["data"]["projects"]:
            slug = project["slug"]
            if slug not in projects:
                continue

            for e in project["localizations"]:
                locale = e["locale"]["code"]
                if locale not in locale_data:
                    locale_data[locale] = {
                        "projects": 0,
                        "missing": 0,
                        "approved": 0,
                        "pretranslated": 0,
                        "total": 0,
                        "completion": 0,
                    }
                locale_data[locale]["missing"] += e["missingStrings"]
                locale_data[locale]["pretranslated"] += e["pretranslatedStrings"]
                locale_data[locale]["approved"] += (
                    e["approvedStrings"] + e["stringsWithWarnings"]
                )
                locale_data[locale]["total"] += e["totalStrings"]
                locale_data[locale]["projects"] += 1
    except Exception as e:
        print(e)

    # Calculate completion percentage
    for locale in locale_data:
        locale_data[locale]["completion"] = round(
            (locale_data[locale]["approved"] / locale_data[locale]["total"]) * 100,
            2,
        )

    # print(json.dumps(locale_data, indent=2))
    diff = list(set(frontend_locales) - set(server_locales))
    if diff:
        print(f"Frontend locales not available in server: {', '.join(diff)}")
    diff = list(set(server_locales) - set(frontend_locales))
    if diff:
        print(f"Server locales not available in frontend: {', '.join(diff)}")

    all_locales = list(set(frontend_locales + server_locales))
    all_locales.sort()
    all_locales.remove("en-US")

    below_locales = []
    missing_locales = []
    threshold = 70
    for loc in all_locales:
        if loc in locale_data:
            if locale_data[loc]["completion"] < threshold:
                below_locales.append(loc)
        else:
            missing_locales.append(loc)

    if missing_locales:
        print(f"Locales missing in Pontoon: {', '.join(missing_locales)}")

    if below_locales:
        print(f"Locales below threshold {threshold}%")
        for loc in below_locales:
            print(f"- {loc}: {locale_data[loc]['completion']}")


if __name__ == "__main__":
    main()