Reproducible benchmark for relocating Elixir Mix build caches across Git worktrees

benchmark_fast_elixir_worktrees.exs

#!/usr/bin/env elixir

defmodule FastWorktreeBenchmark do
  @repo_url "https://github.com/phoenixframework/phoenix.git"
  @commit "0d9c79ebfc0f2065b41e268551638bd8b767eda9"
  @default_trials 5

  def run do
    require_commands!(["git", "mix", "cp"])

    trials =
      System.get_env("BENCHMARK_TRIALS", Integer.to_string(@default_trials))
      |> Integer.parse()
      |> case do
        {value, ""} when value > 0 -> value
        _ -> raise "BENCHMARK_TRIALS must be a positive integer"
      end

    benchmark_root = System.get_env("BENCHMARK_ROOT", System.tmp_dir!())
    File.mkdir_p!(benchmark_root)

    root =
      Path.join(
        benchmark_root,
        "fast-elixir-worktrees-#{System.system_time(:millisecond)}-#{System.unique_integer([:positive])}"
      )

    File.mkdir_p!(root)

    try do
      IO.puts("Benchmark directory: #{root}")
      IO.puts("Measured trials per scenario: #{trials} (after one discarded warmup)\n")

      repo = Path.join(root, "phoenix")
      run!("git", ["clone", "--quiet", @repo_url, repo])
      run!("git", ["checkout", "--quiet", @commit], cd: repo)

      IO.puts("Preparing the default build cache (excluded from measurements)...")
      run!("mix", ["deps.get", "--quiet"], cd: repo)
      run!("mix", ["compile"], cd: repo)

      default_cache = Path.join(root, "cache-default")
      clone_cache!(repo, default_cache, :regular)

      results =
        []
        |> run_scenario(repo, default_cache, root, trials, :fresh)
        |> run_scenario(repo, default_cache, root, trials, :regular_copy)

      IO.puts("\nPreparing a build with check_cwd: false (excluded from measurements)...")
      enable_relocatable_build!(repo)
      run!("git", ["add", "mix.exs"], cd: repo)

      run!(
        "git",
        [
          "-c",
          "user.name=Benchmark",
          "-c",
          "user.email=benchmark@example.com",
          "commit",
          "--quiet",
          "-m",
          "Set check_cwd false"
        ],
        cd: repo
      )

      run!("mix", ["compile"], cd: repo)
      relocatable_commit = output!("git", ["rev-parse", "HEAD"], cd: repo)
      relocatable_cache = Path.join(root, "cache-relocatable")
      clone_cache!(repo, relocatable_cache, :regular)

      results =
        results
        |> run_scenario(
          repo,
          relocatable_cache,
          root,
          trials,
          :relocatable_regular,
          relocatable_commit
        )
        |> run_scenario(
          repo,
          relocatable_cache,
          root,
          trials,
          :relocatable_clone,
          relocatable_commit
        )

      verify!(repo, relocatable_cache, root, relocatable_commit)
      print_report(results, root, relocatable_commit)
    after
      if System.get_env("KEEP_BENCHMARK") == "1" do
        IO.puts("\nKept benchmark directory: #{root}")
      else
        File.rm_rf!(root)
      end
    end
  end

  defp run_scenario(results, repo, cache, root, trials, scenario, commit \\ @commit) do
    IO.puts("\nRunning #{label(scenario)}...")

    measurements =
      for trial <- 0..trials do
        path = Path.join(root, "#{scenario}-#{trial}")

        {worktree_ms, _} =
          timed(fn ->
            run!("git", ["worktree", "add", "--quiet", "--detach", path, commit], cd: repo)
          end)

        {copy_ms, deps_ms, compile_ms, compiled_files} =
          prepare_and_compile!(scenario, cache, path)

        total_ms = worktree_ms + copy_ms + deps_ms + compile_ms

        run_label = if trial == 0, do: "warmup (discarded)", else: Integer.to_string(trial)

        IO.puts(
          "  #{run_label}: #{format(total_ms)} total " <>
            "(worktree #{format(worktree_ms)}, copy #{format(copy_ms)}, " <>
            "deps.get #{format(deps_ms)}, " <>
            "compile #{format(compile_ms)}; #{compiled_files} project files compiled)"
        )

        run!("git", ["worktree", "remove", "--force", path], cd: repo)

        %{
          trial: trial,
          worktree_ms: worktree_ms,
          copy_ms: copy_ms,
          deps_ms: deps_ms,
          compile_ms: compile_ms,
          total_ms: total_ms,
          compiled_files: compiled_files
        }
      end
      |> Enum.reject(&(&1.trial == 0))

    results ++ [%{scenario: scenario, measurements: measurements}]
  end

  defp prepare_and_compile!(:fresh, _cache, path) do
    {deps_ms, _} = timed(fn -> run!("mix", ["deps.get", "--quiet"], cd: path) end)
    {compile_ms, output} = timed(fn -> output!("mix", ["compile", "--verbose"], cd: path) end)

    {0.0, deps_ms, compile_ms, compiled_project_files(output)}
  end

  defp prepare_and_compile!(scenario, cache, path) do
    copy_mode = if scenario == :relocatable_clone, do: :clone, else: :regular

    {copy_ms, _} = timed(fn -> clone_cache!(cache, path, copy_mode) end)
    {deps_ms, _} = timed(fn -> run!("mix", ["deps.get", "--quiet"], cd: path) end)
    {compile_ms, output} = timed(fn -> output!("mix", ["compile", "--verbose"], cd: path) end)

    {copy_ms, deps_ms, compile_ms, compiled_project_files(output)}
  end

  defp clone_cache!(source, destination, mode) do
    for directory <- ["deps", "_build"] do
      source_path = Path.join(source, directory)
      destination_path = Path.join(destination, directory)

      if File.dir?(source_path) do
        File.mkdir_p!(destination_path)

        args =
          case {mode, :os.type()} do
            {:clone, {:unix, :darwin}} -> ["-a", "-c", source_path <> "/.", destination_path]
            {:clone, _} -> ["-a", "--reflink=always", source_path <> "/.", destination_path]
            {:regular, _} -> ["-a", source_path <> "/.", destination_path]
          end

        run_copy!(args, mode, destination)
      end
    end
  end

  defp run_copy!(args, mode, destination) do
    run!("cp", args)
  rescue
    error ->
      if mode == :clone and :os.type() != {:unix, :darwin} do
        raise """
        copy-on-write cloning failed for #{destination}.
        Set BENCHMARK_ROOT to a directory on a reflink-capable btrfs or XFS filesystem.

        #{Exception.message(error)}
        """
      else
        reraise error, __STACKTRACE__
      end
  end

  defp enable_relocatable_build!(repo) do
    path = Path.join(repo, "mix.exs")
    contents = File.read!(path)
    needle = "      elixirc_options: [\n"

    unless String.contains?(contents, needle) do
      raise "could not find Phoenix elixirc_options in #{path}"
    end

    File.write!(
      path,
      String.replace(contents, needle, needle <> "        check_cwd: false,\n", global: false)
    )
  end

  defp verify!(repo, cache, root, commit) do
    IO.puts("\nVerifying the fastest path...")
    path = Path.join(root, "verification")
    run!("git", ["worktree", "add", "--quiet", "--detach", path, commit], cd: repo)
    clone_cache!(cache, path, :clone)

    compile_output = output!("mix", ["compile", "--verbose"], cd: path)

    compiled_paths = compiled_project_file_paths(compile_output)
    compiled_files = length(compiled_paths)
    total_project_files = path |> Path.join("lib/**/*.ex") |> Path.wildcard() |> length()

    if compiled_files >= total_project_files do
      raise "verification rebuilt all #{compiled_files} project files"
    end

    IO.puts("  recompiled #{compiled_files} of #{total_project_files} project files:")
    Enum.each(compiled_paths, &IO.puts("    #{&1}"))

    loaded =
      output!("mix", ["run", "-e", "IO.write(Code.ensure_loaded?(Phoenix.Router))"], cd: path)

    unless loaded == "true" do
      raise "could not load Phoenix.Router from cloned build: #{inspect(loaded)}"
    end

    run!("mix", ["test", "test/phoenix/router/routing_test.exs"], cd: path)
    IO.puts("  #{compiled_files}-file compile, module load, and router tests passed")
    run!("git", ["worktree", "remove", "--force", path], cd: repo)
  end

  defp print_report(results, root, relocatable_commit) do
    baseline = results |> hd() |> median(:total_ms)

    IO.puts("\nResults (median of each scenario)")

    IO.puts(
      "scenario,worktree_ms,copy_ms,deps_ms,compile_ms,total_ms,compiled_files,saved_percent"
    )

    Enum.each(results, fn result ->
      total = median(result, :total_ms)
      saved = (baseline - total) / baseline * 100

      IO.puts(
        Enum.join(
          [
            result.scenario,
            decimal(median(result, :worktree_ms)),
            decimal(median(result, :copy_ms)),
            decimal(median(result, :deps_ms)),
            decimal(median(result, :compile_ms)),
            decimal(total),
            round(median(result, :compiled_files)),
            decimal(saved)
          ],
          ","
        )
      )
    end)

    IO.puts("\nMetadata")
    IO.puts("phoenix_commit=#{@commit}")
    IO.puts("relocatable_commit=#{relocatable_commit}")
    IO.puts("elixir=#{System.version()}")
    IO.puts("otp=#{System.otp_release()}")
    IO.puts("os=#{:os.type() |> inspect()}")
    IO.puts("cpu=#{cpu_model()}")
    IO.puts("filesystem=#{filesystem(root)}")
    IO.puts("benchmark_root=#{Path.dirname(root)}")
  end

  defp median(result, key) do
    values = result.measurements |> Enum.map(&Map.fetch!(&1, key)) |> Enum.sort()
    count = length(values)

    if rem(count, 2) == 1 do
      Enum.at(values, div(count, 2))
    else
      (Enum.at(values, div(count, 2) - 1) + Enum.at(values, div(count, 2))) / 2
    end
  end

  defp compiled_project_files(output) do
    output |> compiled_project_file_paths() |> length()
  end

  # The pinned Mix version emits one `Compiled lib/...` line per project source
  # under `--verbose`. Keep this parser in sync when changing Elixir versions.
  defp compiled_project_file_paths(output) do
    for "Compiled " <> path <- String.split(output, "\n"), String.starts_with?(path, "lib/") do
      path
    end
  end

  defp cpu_model do
    case :os.type() do
      {:unix, :darwin} -> output!("sysctl", ["-n", "machdep.cpu.brand_string"], [])
      {:unix, :linux} -> linux_cpu_model()
      _ -> "unknown"
    end
  end

  defp linux_cpu_model do
    case File.read("/proc/cpuinfo") do
      {:ok, contents} ->
        contents
        |> String.split("\n")
        |> Enum.find_value("unknown", fn line ->
          case String.split(line, ":", parts: 2) do
            ["model name" <> _, value] -> String.trim(value)
            _ -> nil
          end
        end)

      _ ->
        "unknown"
    end
  end

  defp filesystem(root) do
    case :os.type() do
      {:unix, :darwin} ->
        device =
          "df"
          |> output!(["-P", root], [])
          |> String.split("\n")
          |> List.last()
          |> String.split()
          |> hd()

        "diskutil"
        |> output!(["info", device], [])
        |> String.split("\n")
        |> Enum.find_value("unknown", fn line ->
          case String.split(line, ":", parts: 2) do
            [key, value] ->
              if String.trim(key) == "File System Personality", do: String.trim(value)

            _ ->
              nil
          end
        end)

      {:unix, :linux} ->
        output!("stat", ["-f", "-c", "%T", root], [])

      _ ->
        "unknown"
    end
  end

  defp label(:fresh), do: "fresh worktree + deps.get + compile"
  defp label(:regular_copy), do: "regular deps + _build copy"
  defp label(:relocatable_regular), do: "check_cwd: false + regular copy"
  defp label(:relocatable_clone), do: "check_cwd: false + copy-on-write clone"

  defp timed(fun) do
    started = System.monotonic_time()
    result = fun.()
    elapsed = System.monotonic_time() - started
    {System.convert_time_unit(elapsed, :native, :microsecond) / 1_000, result}
  end

  defp format(milliseconds), do: :erlang.float_to_binary(milliseconds / 1_000, decimals: 3) <> "s"
  defp decimal(number), do: :erlang.float_to_binary(number, decimals: 2)

  defp output!(command, args, opts) do
    {output, status} = System.cmd(command, args, Keyword.merge([stderr_to_stdout: true], opts))

    if status != 0 do
      raise "command failed (#{status}): #{command} #{Enum.join(args, " ")}\n#{output}"
    end

    String.trim(output)
  end

  defp run!(command, args, opts \\ []) do
    _ = output!(command, args, opts)
    :ok
  end

  defp require_commands!(commands) do
    Enum.each(commands, fn command ->
      unless System.find_executable(command), do: raise("missing command: #{command}")
    end)
  end
end

FastWorktreeBenchmark.run()