Sync versions from yarn.lock back into package.json

yarn-sync.js

const fs = require("fs");

/**
 * Removes matching outer quotes from a string.
 *
 * @param {string} text - String to unquote
 * @returns {string} - Unquoted string
 */
const unquote = text => /(["'])?(.*)\1/.exec(text)[2];

/**
 * @typedef {object} YarnLockItem
 * @property {string[]} pkgs - Array of package version specs, e.g. "yippee-ki-yay@^1.23.45", "@foo/bar@^1.6.0"
 * @property {string} version - Installed version
 * @property {string} resolved - Resolved package URL
 * @property {string} integrity - Package integrity
 * @property {object.<string, string>} dependencies - Package dependencies and their version specs
 */

/**
 * Extracts information about installed packages from yarn.lock file.
 *
 * NB: functionality for parsing a yarn.lock file exists in a package called `@yarnpkg/lockfile`,
 * however this package pulls in way too many dependencies (all of yarn, it seems).
 *
 * @param {string} filename - Path to yarn.lock file
 * @returns {object.<string, YarnLockItem>} Installed package information, keyed by package version spec
 */
const parseYarnLockFile = s => {
  const lines = s.replace(/\r/g, "").split(/\n/);

  const entries = {};
  let entry;
  let key;

  lines.forEach(line => {
    const indent = /^(\s*)/.exec(line)[1].length;
    line = line.trim();

    if (line === "") {
      if (entry) {
        // Add an entry for each of the package specs in the item
        entry.pkgs.forEach(pkg => {
          entries[pkg] = entry;
        });
        entry = null;
      }
    } else if (line[0] === "#") {
      // Comment, skip
    } else if (indent === 0) {
      // Start of entry
      entry = {
        // Remove trailing colon, split, trim and unquote
        pkgs: line
          .replace(/:$/, "")
          .split(",")
          .map(s => unquote(s.trim()))
      };
    } else if (indent === 2) {
      let match;
      if ((match = /^(\w+) (.+)/.exec(line))) {
        entry[match[1]] = unquote(match[2]);
      } else if ((match = /^(\w+):$/.exec(line))) {
        key = match[1];
        entry[key] = {};
      }
    } else if (indent === 4) {
      const match = /^(.+) (.+)/.exec(line);
      if (match) {
        entry[key][unquote(match[1])] = unquote(match[2].trim());
      }
    } else {
      console.warn("Line not understood:", line);
    }
  });

  return entries;
};

const updatePackageJson = (packageJson, yarnLock) => {
  let changeCount = 0;

  const updateSection = sectionName => {
    console.log("Updating", sectionName, "...");

    const section = packageJson[sectionName];

    Object.entries(section).forEach(([pkg, versionSpec]) => {
      const dependency = `${pkg}@${versionSpec}`;

      // Get the version spec prefix, e.g. '^' or '>=', or none.
      // We support version specs containing a single semver, other types of version spec are untested a.t.m.
      // (version spec format is documented here: https://docs.npmjs.com/files/package.json#dependencies)
      const versionSpecPrefix = /^([^\d]*)/.exec(versionSpec)[1];

      const yarnLockEntry = yarnLock[dependency];
      if (yarnLockEntry) {
        const actualVersion = yarnLockEntry.version;
        const actualVersionSpec = `${versionSpecPrefix}${actualVersion}`;

        if (actualVersionSpec !== versionSpec) {
          console.log("  Updating:", dependency, "=>", actualVersionSpec);
          section[pkg] = actualVersionSpec;
          ++changeCount;
        } else {
          console.log("  Up-to-date:", dependency);
        }
      } else {
        console.warn("  !!! Missing yarn.lock entry for:", dependency);
      }
    });

    console.log("  Done.");
  };

  [
    "dependencies",
    "devDependencies",
    "optionalDependencies",
    "peerDependencies"
  ].forEach(sectionName => {
    if (sectionName in packageJson) {
      updateSection(sectionName);
    }
  });

  return changeCount;
};

const main = () => {
  console.log("Reading package.json ...");
  const packageJsonFile = fs.readFileSync("package.json", "utf8");
  const packageJson = JSON.parse(packageJsonFile);

  console.log("Reading yarn.lock ...");
  const yarnLockFile = fs.readFileSync("yarn.lock", "utf8");
  const yarnLock = parseYarnLockFile(yarnLockFile);

  const changeCount = updatePackageJson(packageJson, yarnLock);

  if (changeCount > 0) {
    const outFilename = "package_synced.json";
    console.log("Writing changes to:", outFilename);
    fs.writeFileSync(outFilename, JSON.stringify(packageJson, null, 2));
  } else {
    console.log("No changes");
  }
};

main();

yarn-sync.py

#!/usr/bin/env python3

from collections import OrderedDict
import json
import re
from typing import List, Dict


# class YarnLockItem:
#     pkgs: List[str]
#     version: str
#     resolved: str
#     integrity: str
#     dependencies: Dict[str, str]

#     def __init__(self, pkgs):
#         self.pkgs = pkgs


def unquote(text):
    return re.sub(r"([\"'])?(.*)\1", r"\2", text)


def load_package_json_file(fp):
    return json.load(fp, object_pairs_hook=OrderedDict)


def write_package_json_file(package_json, fp):
    json.dump(package_json, fp, indent=2, ensure_ascii=False)
    fp.write("\n")


def load_yarn_lock_file(fp):
    entries = {}
    entry = None
    key = None

    for line in f:
        indent = len(re.match(r"^(\s*)", line).group(0))
        line = line.strip()

        if not line:
            if entry:
                # Add an entry for each of the package specs in the item
                for pkg in entry["pkgs"]:
                    entries[pkg] = entry
                entry = None
        elif line[0] == "#":
            # Comment, skip
            pass
        elif indent == 0:
            # Start of entry
            entry = {
                "pkgs": ([unquote(pkg.strip()) for pkg in line.rstrip(":").split(",")])
            }
        elif indent == 2:
            match = re.match(r"^(?P<key>\w+) (?P<value>.+)", line)
            if match:
                entry[match.group("key")] = unquote(match.group("value"))
            else:
                match = re.match(r"(?P<key>\w+):$", line)
                if match:
                    key = match.group("key")
                    entry[key] = {}
        elif indent == 4:
            match = re.match(r"^(?P<key>.+) (?P<value>.+)", line)
            if match:
                entry[key][unquote(match.group("key"))] = unquote(
                    match.group("value").strip()
                )

    return entries


def update_package_json(package_json, yarn_lock):
    change_count = 0

    def update_section(section_name):
        print("Processing", section_name, "...")

        section = package_json[section_name]

        for pkg, version_spec in section.items():
            dependency = f"{pkg}@{version_spec}"
            yarn_lock_entry = yarn_lock.get(dependency)
            if yarn_lock_entry:
                # Get the version spec prefix, e.g. '^' or '>=', or none.
                # Only single version semver values are currently tested
                # (version spec format is documented here: https://docs.npmjs.com/files/package.json#dependencies)
                version_spec_prefix = re.match(r"([^\d]*)", version_spec).group(0)

                actual_version = yarn_lock_entry["version"]
                actual_version_spec = f"{version_spec_prefix}{actual_version}"

                if actual_version_spec != version_spec:
                    print(
                        "  Updating:", dependency, "=>", actual_version_spec,
                    )
                    section[pkg] = actual_version_spec
                    change_count += 1
                else:
                    print("  Up-to-date:", dependency)
            else:
                print(
                    "  !!! Missing yarn.lock entry for:", dependency,
                )

        print("  Done.")

    for section_name in [
        "dependencies",
        "devDependencies",
        "optionalDependencies",
        "peerDependencies",
    ]:
        if section_name in package_json:
            update_section(section_name)

    return change_count


if __name__ == "__main__":
    print("Reading package.json ...")
    with open("package.json", "rt") as f:
        package_json = load_package_json_file(f)

    print("Reading yarn.lock ...")
    with open("yarn.lock", "rt") as f:
        yarn_lock = load_yarn_lock_file(f)

    change_count = update_package_json(package_json, yarn_lock)

    if change_count > 0:
        print("Writing changes to package_synced.json ...")
        with open("package_synced.json", "wt") as f:
            write_package_json_file(package_json, f)
    else:
        print("No changes")

Thank you so much to this!

I just had to fix line 103 of the JS script, changing section[dependency] = actualVersionSpec; to section[pkg] = actualVersionSpec; to get it working perfectly for my project.

Good catch, thanks! Fixed the gist, and I am glad to see others can benefit from my contributions here :)

Thanks for sharing! I have a monorepo with multiple libs like this:

{
	"workspaces": [
		"apps/*",
		"libs/*"
	]
}

Are you considering modifying the script to support workspaces? 🙃

I haven't used yarn in a long time, but feel free to contribute!