nix test script for the LLVM package set

llvm-pkgset-test.nix

{ nixpkgs ? ./.
, lib ? import ./lib

# Host systems to test from.
, systems ? [
    "i686-linux"

    "x86_64-linux"
    # "x86_64-darwin"

    "aarch64-linux"
    # "aarch64-darwin"

    "armv7l-linux" # I don't have a machine capable of running 32-bit arm code at the moment..
  ]

# This controls whether to test cross-compiling (both cross-compiling LLVM _for_
# platforms and cross-compiling `testStdenvPackages` _using_ LLVM).
#
# For now, we only testing cross-compiling from an `x86_64` host.
, testCross ? { buildPlatform, hostPlatform, targetPlatform }:
  buildPlatform.system == "x86_64-linux"
  # true # !!! it's good to check that this evals successfully but be warned:
         # instantiating the full matrix of
         # `systems x (crossSystems + crossTargetOnlySystems)` nixpkgs instances
         # will use a fair bit of RAM

# We test both that we can build LLVM for these platforms and that we can use
# LLVM to build packages for these platforms.
#
# These should be attributes in `pkgsCross`/`lib.systems.examples`.
, crossSystems ? [
    "gnu64"
    "musl64"

    # TODO: x86 darwin, arm64 darwin
    # TODO: aarch64-linux
    # TODO: ppc

    "riscv64" # "riscv64-linux"
    "armv7l-hf-multiplatform"

    # TODO: add static configurations too? nah

    # "x86_64-freebsd" # !!! See below.
  ]

# TODO: Maybe just use `release-cross.nix`..

# Extra platforms that we build _for_ using LLVM but don't build LLVM for
# (i.e. it doesn't make sense to build LLVM for bare-metal targets but we can
# use LLVM as part of the toolchain that builds packages for such targets).
, crossTargetOnlySystems ? [
    # TODO: arm embedded, mips, iphone, android, s390
    "wasi32"
    "mingw32"
    "mingwW64"       # TODO: move to `crossSystems`, we should be able to
                     # compile LLVM for windows..
    "x86_64-freebsd" # !!! we should be able to build LLVM for freebsd;
                     # currently the blocker is that most of the LLVM package
                     # set  depends directly on `libxml2` as a runtime dep which
                     # needs python which isn't marked as supported on freebsd
                     #
                     # to fix we should either get python to work or look into
                     # whether the LLVM packages actually need the python
                     # functionality in libxml2 (for `llvm`, `lld`, `clang`:
                     # almost certainly not. not sure about `lldb`)

  ]

, llvmPkgSetName ? "llvmPackages_15"

# Packages to try to build with the llvm stdenv we build.
#
# Packages specified here are built for:
#   - the native system + the native system with `pkgsStatic`, `pkgsMusl`
#   - crossSystems
#   - crossTargetOnlySystems
#
# Each element of this list can be:
#   - a string specifying a path (i.e. `"python3.pkgs.psutil"`)
#   - a list specifying a path (i.e. `["python3" "pkgs" "psutil"]`)
#   - a function taking the build, host, and target platforms that then returns
#     one of the above or null; for example:
#     ```nix
#     { buildPlatform, hostPlatform, targetPlatform }:
#       if targetPlatform.isLinux then
#         [ "python3" "pkgs" "psutil" ]
#       else null;
#     ```
, testStdenvPackages ? [
  "hello"
]
}: let
  # TODO: add enablePolly, enableZ3, other options?
  #
  # OTOH, most of these if not all are enabled by default where supported..

  collectDrvsAndTests = pkgSet: let
    processSet = set: lib.foldl' lib.mergeAttrs {} (
      lib.pipe set [
        (lib.mapAttrsToList (name: value: { inherit name value; }))

        (builtins.map ({ name, value }: let
          orig = lib.optionalAttrs (lib.isDerivation value) { ${name} = value; };

          tests = if value ? tests then
            lib.mapAttrs' (testName: value: {
              # `nix-build` seems to ignore attrnames containing `/` and `.` and
              # most other symbols
              name = "${name}__test_${testName}";
              inherit value;
            }) value.tests
          else {};

          children = lib.optionalAttrs (value.recurseForDerivations or false) (
            lib.mapAttrs' (childName: childValue: let parentName = name; in {
              # `nix-build` seems to ignore attrnames containing `/` and `.`
              name = "${parentName}__${childName}";
              value = childValue;
            }) (processSet value)
          );
        in
          (orig // tests // children)
        ))
      ]
    );

    result = (lib.optionalAttrs (lib.isDerivation pkgSet) { main = pkgSet; }) //
      (processSet pkgSet);
  in lib.recurseIntoAttrs result;

  ## Test that building LLVM works (regular and weirder configurations like musl and cross compiling):
  # llvmPackages_15.* (recurse for derivations)
  # llvmPackages_15.* pass thru tests
  #
  # pkgsStatic.llvmPackages_15.*, pass thru tests
  # pkgsMusl.llvmPackages_15.*, pass thru tests
  #
  # # Test that we can cross compile LLVM for other platforms:
  # pkgsCross.<systems + crossSystems>.llvmPackages_15.*, pass thru tests
  #
  ## Test that the LLVM stdenv works (both for native compilation and cross-compilation):
  # pkgsLLVM.hello # with overlay setting llvmPackages to llvmPackages_15; to test that the stdenv works..
  # pkgsLLVM.pkgsStatic.hello
  # pkgsLLVM.pkgsMusl.hello
  #
  # # Test that we can _use_ LLVM to cross compile for other platforms:
  # (nixpkgs { overlays = ...; crossSystem = sys // { useLLVM = true; }; }).hello
in let
  nixpkgsInstances = builtins.listToAttrs (
    builtins.map
    (system: {
      name = system;
      value = import nixpkgs { inherit system; };
    })
    systems
  );

  llvmOverlay = (f: p: { llvmPackages = f.${llvmPkgSetName}; });

  nixpkgsInstancesLLVMOverlay = builtins.listToAttrs (
    builtins.map
    (system: {
      name = system;
      value = import nixpkgs {
        inherit system;
        overlays = [ llvmOverlay ];
      };
    })
    systems
  );

  getPackages = {
    np,
    packagesByPath,
    prefix ? "",

    # Each element must be a string in `pkgsCross` or must be an attrset with
    # `crossSystemName` and `pkgSet` where `pkgSet` is a nixpkgs instance.
    crossSystems ? [],

    crossSystemFilter ? testCross,
  }: let
    plat = np.targetPlatform;

    pkgPathToAttrName = pkgPath: if lib.isList pkgPath then
      lib.concatStringsSep "__" pkgPath
    else
      lib.replaceStrings ["."] ["__"] pkgPath;
    pkgPathLookup = pkgPath: set: if lib.isList pkgPath then
      lib.getAttrFromPath pkgPath set
    else
      pkgPathLookup (lib.splitString "." pkgPath) set;

    extractPackagesByPath = pkgset: list:
      builtins.listToAttrs (builtins.map (pkgPath: {
        name = pkgPathToAttrName pkgPath;
        value = (pkgPathLookup pkgPath pkgset) // {
          # for debugging:
          __origin = {
            inherit pkgPath pkgset;
          };
        };
      }) list);

    getPackages = pkgset: let
      pkgs = lib.pipe packagesByPath [
        # Invoke any functions:
        (builtins.map
          (p: if lib.isFunction p then
            p { inherit (pkgset) buildPlatform hostPlatform targetPlatform; }
          else p)
        )
        # Filter out nulls:
        (builtins.filter (p: p != null))
      ];
    in lib.recurseIntoAttrs (extractPackagesByPath pkgset pkgs);
  in lib.recurseIntoAttrs {
    ${ prefix + "native" } = getPackages np;
    ${ prefix + "static" } = getPackages np.pkgsStatic;

    # Gate `musl` on Linux + specific host archs:
    #
    # As of this writing, the musl stdenv isn't supported on `i686`:
    # https://github.com/NixOS/nixpkgs/blob/3586ad34ff9227542c13909ce94785f9778eb2fe/pkgs/stdenv/linux/default.nix#L23-L27
    ${ if (plat.isLinux && (!plat.isx86_32)) then prefix + "musl" else null } =
      getPackages np.pkgsMusl;

    # Cross:
    ${ prefix + "cross" } = let
      getCrossPackageSet = crossSys: if lib.isAttrs crossSys then
        assert ((crossSys ? crossSystemName) && crossSys ? pkgSet); crossSys
      else
        { crossSystemName = crossSys; pkgSet = np.pkgsCross.${crossSys}; };

      crossSets = lib.pipe crossSystems [
        # Get the cross package set (either `pkgsCross` or provided explicitly):
        (builtins.map getCrossPackageSet)
        # Filter using `crossSystemFilter`:
        (builtins.filter
          ({ pkgSet, ... }: crossSystemFilter {
            inherit (pkgSet) buildPlatform hostPlatform targetPlatform;
          })
        )
        # Get the packages to test out of any remaining package sets:
        (builtins.map ({ crossSystemName, pkgSet }: {
          name = crossSystemName;
          value = getPackages pkgSet;
        }))
        # Turn into an attrset:
        builtins.listToAttrs
      ];

    in lib.recurseIntoAttrs crossSets;
  };

  tests = builtins.listToAttrs (builtins.map (system: {
    name = system;
    value = let
      np = nixpkgsInstances.${system};
      npLlvmStdenv = nixpkgsInstancesLLVMOverlay.${system};

      ### Test that we can build LLVM ###
      # native, native+static, native+musl, building LLVM for cross systems:
      buildLlvm = getPackages {
        inherit np;
        packagesByPath = [ llvmPkgSetName ];
        inherit crossSystems;
      };

      ### Test that we can build *using* LLVM ###
      # for native, native+static, native+musl
      # and building for cross systems using LLVM
      buildUsingLlvm = getPackages {
        np = npLlvmStdenv.pkgsLLVM;
        packagesByPath = testStdenvPackages;

        # To build for a cross system using LLVM we either have to do:
        # `pkgsCross.${crossSys}.pkgsLLVM`
        #
        # Or we can set up a nixpkgs instance for our cross target with `useLLVM
        # = true`:
        crossSystems = let
          systems = crossSystems ++ crossTargetOnlySystems;
          nixpkgsCrossInstances = builtins.map (sys: {
            crossSystemName = sys;
            pkgSet = import nixpkgs {
              inherit system;
              crossSystem = lib.systems.examples.${sys} // { useLLVM = true; };
              overlays = [ llvmOverlay ];
            };
          }) systems;
        in nixpkgsCrossInstances;
      };
    in lib.recurseIntoAttrs {
      inherit buildLlvm buildUsingLlvm;
    };
  }) systems);

  log = pkgset: let
    str = builtins.toJSON pkgset;
  in builtins.trace str pkgset;

in (collectDrvsAndTests tests) // {
  inherit tests; /* for poking around in the repl */
}