Requires a running instance of WebDriver on port 4444 (and ImageMagick)

run_svg_tests.py

#!/usr/bin/env python3

from collections import namedtuple
from pathlib import Path
import base64
import httplib2
import json
import os
import subprocess

TestInstance = namedtuple('TestInstance', ['input', 'expected'])
github_bearer_token = "bearer ASDF1234FDSA4321"


class WebDriver:
    baseUrl = "http://0.0.0.0:4444"
    sessionId = None

    def makeGetRequest(self, url):
        _, content = httplib2.Http().request(self.baseUrl + url)
        return content.decode("utf-8")

    def makePostRequest(self, url, data):
        _, content = httplib2.Http().request(self.baseUrl + url, "POST", body=json.dumps(data), headers={"Content-Type": "application/json"})
        return content.decode("utf-8")

    def createSession(self):
        session = self.makePostRequest("/session", {
            "capabilities": {
                "alwaysMatch": {
                    "serenity:ladybird": {
                        "headless": True,
                    }
                }
            }
        })
        self.sessionId = json.loads(session)["value"]["sessionId"]

    def setWindowSize(self, width, height):
        self.makePostRequest("/session/" + self.sessionId + "/window/rect", {
            "width": width,
            "height": height
        })

    def navigateTo(self, url):
        self.makePostRequest("/session/" + self.sessionId + "/url", {
            "url": url
        })

    def takeScreenshot(self):
        screenshot = self.makeGetRequest("/session/" + self.sessionId + "/screenshot")
        screenshotBase64 = json.loads(screenshot)["value"]
        return base64.b64decode(screenshotBase64)


def loadTests(root):
    tests = []

    def handleDirectory(data):
        for entry in data:
            if entry["type"] == "file":
                if entry["name"].endswith(".svg"):
                    svgPath = entry["download_url"]
                    pngPath = svgPath.replace(".svg", ".png")
                    tests.append(TestInstance(svgPath, pngPath))
            elif entry["type"] == "dir":
                print("Loading tests in " + entry["path"])

                _, content = httplib2.Http(cache="github-cache").request(entry["url"], "GET", headers={"Authorization": github_bearer_token})
                handleDirectory(json.loads(content))

    _, content = httplib2.Http(cache="github-cache").request(root, "GET", headers={"Authorization": github_bearer_token})
    handleDirectory(json.loads(content))

    return tests


def screenshotUrl(url, saveTo):
    driver.navigateTo(url)

    screenshotBytes = driver.takeScreenshot()
    with open(saveTo, "wb") as f:
        f.write(screenshotBytes)


def generateComparison(inputFile, outputFile, diffType):
    diffFile = str(outputFile).replace(".png", "." + diffType + ".png")
    result = subprocess.run(["compare", "-metric", diffType, inputFile, outputFile, diffFile], capture_output=True, text=True)
    return result.stderr

def runTest(test, only_analyze=False):
    # Ensure output folder exists and setup variables
    baseUrl = "/main/tests/"
    testPath = test.input[test.input.index(baseUrl) + len(baseUrl):]
    basePath = "Tests/resvg/" + testPath
    filename = test.input.split("/")[-1]
    saveTo = '/'.join(basePath.split("/")[0:-1])
    saveTo = saveTo.replace(baseUrl, "")

    svgFile = Path(saveTo + "/" + filename).absolute()
    inputFile = Path(saveTo + "/" + filename.replace(".svg", ".ladybird.png")).absolute()
    outputFile = Path(saveTo + "/" + filename.replace(".svg", ".png")).absolute()
    os.makedirs(saveTo, exist_ok=True)

    print("Running test for " + test.input)
    
    # Download SVG file if it doesnt exist
    if not os.path.exists(svgFile):
        _, content = httplib2.Http().request(test.input)
        with open(svgFile, "wb") as f:
            f.write(content)

    # Screenshot rendered SVG file if it doesnt exist
    if not os.path.exists(inputFile) or not only_analyze:
        screenshotUrl(test.input,  inputFile)

    # Screenshot expected PNG file if it doesnt exist
    # NOTE: We need to screenshot the PNG to ensure the transparency is the same
    # FIXME: Allow transparent windows?
    if not os.path.exists(outputFile):
        screenshotUrl(test.expected,  outputFile)

    # Get difference metrics
    SSIMScore = generateComparison(inputFile, outputFile, "SSIM")
    AEScore = (500 * 500) - int(generateComparison(inputFile, outputFile, "AE"))
    
    print("Similarity score for " + testPath + ": " + SSIMScore)
    
    return {
        "file": testPath,
        "SSIM": SSIMScore,
        "AE": AEScore
    }


if __name__ == "__main__":
    driver = WebDriver()

    # Start a new session with the correct screen size
    driver.createSession()
    driver.setWindowSize(500, 500)

    tests = loadTests("https://api.github.com/repos/linebender/resvg-test-suite/contents/tests")

    # Reset results file
    if os.path.exists("Tests/resvg/results.json"):
        os.remove("Tests/resvg/results.json")

    crashingTests = [
        "https://raw.githubusercontent.com/linebender/resvg-test-suite/main/tests/masking/clipPath/recursive-on-child.svg",
        "https://raw.githubusercontent.com/linebender/resvg-test-suite/main/tests/masking/clipPath/self-recursive.svg",
        "https://raw.githubusercontent.com/linebender/resvg-test-suite/main/tests/masking/mask/recursive-on-child.svg",
        "https://raw.githubusercontent.com/linebender/resvg-test-suite/main/tests/masking/mask/recursive-on-self.svg",
        "https://raw.githubusercontent.com/linebender/resvg-test-suite/main/tests/masking/mask/recursive.svg",
        "https://raw.githubusercontent.com/linebender/resvg-test-suite/main/tests/masking/mask/self-recursive.svg",
        "https://raw.githubusercontent.com/linebender/resvg-test-suite/main/tests/structure/use/self-recursive.svg",
    ]
    
    results = []
    
    for test in tests:
        if test.input in crashingTests:
            print("Skipping test " + test.input + " as it is marked as crashing")
            continue
        
        result = runTest(test, only_analyze=True)
        results.append(result)
        
    json.dump(results, open("Tests/resvg/results.json", "w"), indent=4)