Rudimentary visual regression testing strategery

README.md

Introduction

This gist documents the setup I originally used to run visual diffs on the MDC Web demo pages for testing material-components/material-components-web#1285, but I have used it since then on other projects.

Setup

1. Create a directory containing the js and txt files from this gist
2. npm i

Process

1. In your code checkout, switch to the branch you want to compare against, e.g. main, and run the dev server
2. If you've run these scripts before, run rm -r ss-* in the directory to clean out files from the previous run
3. In the folder with these files, run node puppeteer which will produce a ss subdirectory
4. mv ss ss-old
5. In your code checkout, switch to the branch with the changes to test. Depending on whether there are significant changes e.g. added files, you may want to shut down the dev server before switching branch and restart it after.
6. In the folder with these files, run node puppeteer which will again produce a ss subdirectory
7. mv ss ss-new
8. Run node blinkdiff, which will produce a ss-diff subdirectory and output results to stdout

The blinkdiff.js script will output any notable results (any pages with any changes, regardless of whether they're over or under threshold) first, followed by a list of the files that had no observed changes.

For any files with noted changes, you can look at ss-diff/<filename>.png to see a visual diff (left = old, right = new, middle highlights what changed).

You can see an example of what a diff looks like in material-components/material-components-web#1286.

blinkdiff.js

const BlinkDiff = require('blink-diff');
const fs = require('fs');
const path = require('path');
const images = fs.readdirSync(path.join(__dirname, 'ss-old'));
const passedWithNoDiffs = [];

const SCREENSHOT_DIFF_DIR = path.join(__dirname, 'ss-diff');

console.log(`Comparing ${images.length} pairs of images...`);

if (!fs.existsSync(SCREENSHOT_DIFF_DIR)) {
  fs.mkdirSync(SCREENSHOT_DIFF_DIR);
}

let ran = 0;
for (const pngName of images) {
  var diff = new BlinkDiff({
    imageAPath: path.join(__dirname, 'ss-old', pngName),
    imageBPath: path.join(__dirname, 'ss-new', pngName),

    thresholdType: BlinkDiff.THRESHOLD_PERCENT,
    threshold: 0.01, // 1% threshold

    imageOutputPath: path.join(SCREENSHOT_DIFF_DIR, pngName)
  });

  diff.run(function (error, result) {
    if (error) {
      console.error('ERR', pngName, error);
    } else {
      if (diff.hasPassed(result.code) && !result.differences) {
        passedWithNoDiffs.push(pngName);
      } else {
        console.log(pngName, diff.hasPassed(result.code) ? 'Passed' : 'Failed', `(${result.differences} differences)`);
      }
    }

    if (++ran === images.length) {
      console.log('--- The following passed with 0 differences ---');
      passedWithNoDiffs.forEach((entry) => console.log(entry));
    }
  });
}

package.json

{
  "name": "screenshots",
  "devDependencies": {
    "blink-diff": "1.0.13",
    "puppeteer": "1.19.0"
  }
}

puppeteer.js

const puppeteer = require('puppeteer');
const fs = require('fs');
const path = require('path');

const SCREENSHOT_DIR = path.join(__dirname, 'ss');

async function run() {
  if (!fs.existsSync(SCREENSHOT_DIR)) {
    fs.mkdirSync(SCREENSHOT_DIR);
  }

  const urlsFile = process.argv[2] || 'urls.txt';
  // Ignore lines beginning with "# " (don't ignore just # in case of hash-routed apps...)
  const lines = fs.readFileSync(path.join(__dirname, urlsFile), 'utf8').split(/[\r\n]+/).filter((url) => url && url.slice(0, 2) != '# ');

  let urlPrefix = 'http://localhost:8080/';
  if (lines[0].indexOf('http') === 0) {
    urlPrefix = lines.shift();
  }

  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  for (let i = 0; i < lines.length; i++) {
    const [url, width = '800'] = lines[i].split(/\s+/);
    const widths = width.split(',');
    for (let j = 0; j < widths.length; j++) {
      await page.setViewport({width: +widths[j], height: 600});
      await page.goto(urlPrefix + url);
      await page.screenshot({
        fullPage: true,
        path: path.join(SCREENSHOT_DIR, `${url.replace(/[\/#]/g, '-')}-${widths[j]}.png`)
      });
      console.log(`[Done] ${url} @ ${widths[j]}`);
    }
  }

  await browser.close();
}

run().catch(console.error.bind(console));

urls.txt

# Notes on this file's format:
# You can prefix lines with "# " (note the space) to have the scripts ignore them.
# You can optionally include a base URL at the top of this file, which all other lines will be relative to.
# You can optionally provide a comma-delimited list of widths to test after the filename (default is 800)
# e.g. "button.html 375,552,728,904,1080,1440"
button.html
card.html
checkbox.html
dialog.html
drawer/permanent-drawer-above-toolbar.html
drawer/permanent-drawer-below-toolbar.html
drawer/persistent-drawer.html
drawer/temporary-drawer.html
elevation.html
fab.html
grid-list.html
icon-toggle.html
index.html
layout-grid.html
linear-progress.html
list.html
radio.html
ripple.html
select.html
simple-menu.html
slider.html
snackbar.html
switch.html
tabs.html
textfield.html
theme.html
typography.html
toolbar/index.html