NixOS Test Harvester

README.md

NixOS Test Harvester

Exposes a library helper and a flake-parts module to dynamically inspect NixOS configurations and register their enabled services to Nixpkgs' integration tests for CI/checks.

How to use

Flake Input

inputs.nixos-test-harvester = {
  url = "https://gist.github.com/Malix-Labs/d58d2af9be08519099b17abb25f4d5be";
  type = "git";
  inputs.nixpkgs.follows = "nixpkgs";
  # inputs.flake-parts.follows = "flake-parts"; # Optional, only if using flake-parts module
};

NixOS Module (Importing in host config)

Import the module and enable it. Optionally configure the filters and nesting:

imports = [
  inputs.nixos-test-harvester.nixosModules.default
];

services.nixos-test-harvester = {
  enable = true;
  
  filter.heavy.enable = false; # Don't filter out heavy system tests (defaults to true)
  filter.custom = [ "service-to-ignore" "another-one" ]; # Custom service names to filter out
  # filter.hydra = false; # Don't filter out tests already passed by Hydra (defaults to true)

  nesting.enable = true; # Group all tests under a single linkFarm check
  # nesting.name = "nixos-tests-custom"; # Optional custom top check name
};

Flake Checks (Harvesting the tests)

Using flake-parts

Import the flake-parts module to automatically populate checks:

imports = [
  inputs.nixos-test-harvester.flakeModules.default
];

Standard Flake

Or pass your hosts configurations and library to harvestAll:

checks = inputs.nixos-test-harvester.lib.harvestAll {
  hosts = self.nixosConfigurations;
  inherit (inputs.nixpkgs) lib;
};

flake.lock

{
  "nodes": {
    "flake-parts": {
      "inputs": {
        "nixpkgs-lib": [
          "nixpkgs"
        ]
      },
      "locked": {
        "lastModified": 1782949081,
        "narHash": "sha256-vp6Y/Grm98ESt6ceOkWiHWyZRDV3J1RID4w+6NWK9yA=",
        "owner": "hercules-ci",
        "repo": "flake-parts",
        "rev": "17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e",
        "type": "github"
      },
      "original": {
        "owner": "hercules-ci",
        "repo": "flake-parts",
        "type": "github"
      }
    },
    "nixpkgs": {
      "locked": {
        "lastModified": 1782614948,
        "narHash": "sha256-ePjCwr1sNm9NYUqywL7QfK3JnlS015msC+eBu2zKlp8=",
        "owner": "nix-community",
        "repo": "nixpkgs.lib",
        "rev": "db3f255737b94216eb71cce308e2912cf6bc2d7c",
        "type": "github"
      },
      "original": {
        "owner": "nix-community",
        "repo": "nixpkgs.lib",
        "type": "github"
      }
    },
    "options-schema": {
      "inputs": {
        "nixpkgs": [
          "nixpkgs"
        ]
      },
      "locked": {
        "lastModified": 1783014457,
        "narHash": "sha256-Rbos5/gQCfwcQhjqR0zSbN2VSViwqfMQeEFyVG5n93U=",
        "ref": "refs/heads/main",
        "rev": "2f46704b70c59916ca58e69c1534ef0608388be5",
        "revCount": 1,
        "type": "git",
        "url": "https://gist.github.com/Malix-Labs/eec30a668f03ca4ad1060c68c3f5c442"
      },
      "original": {
        "type": "git",
        "url": "https://gist.github.com/Malix-Labs/eec30a668f03ca4ad1060c68c3f5c442"
      }
    },
    "root": {
      "inputs": {
        "flake-parts": "flake-parts",
        "nixpkgs": "nixpkgs",
        "options-schema": "options-schema"
      }
    }
  },
  "root": "root",
  "version": 7
}

flake.nix

{
  description = "NixOS Test Harvester";

  inputs = {
    nixpkgs.url = "github:nix-community/nixpkgs.lib";
    options-schema = {
      url = "https://gist.github.com/Malix-Labs/eec30a668f03ca4ad1060c68c3f5c442";
      type = "git";
      inputs.nixpkgs.follows = "nixpkgs";
    };
    flake-parts = {
      url = "github:hercules-ci/flake-parts";
      inputs.nixpkgs-lib.follows = "nixpkgs";
    };
  };

  outputs =
    {
      options-schema,
      nixpkgs,
      ...
    }:
    let
      inherit (nixpkgs) lib;

      genAttrsNull = list: lib.genAttrs list (_: null);

      options = {
        enable = lib.mkEnableOption "NixOS test harvester";
        filter = {
          custom = lib.mkOption {
            type = lib.types.listOf lib.types.str;
            default = [ ];
            description = "Custom service names to filter out from harvested tests.";
          };
          heavy = {
            enable = lib.mkEnableOption "filtering out heavy nixos tests" // {
              default = true;
            };
            list = lib.mkOption {
              type = lib.types.listOf lib.types.str;
              default = [
                "libinput"
                "udisks2"
                "logrotate"
                "nscd"
                "geoclue2"
                "accounts-daemon"
                "dbus"
                "upower"
              ];
              description = "Services considered heavy and excluded when filter.heavy.enable is true.";
            };
          };
          hydra = lib.mkEnableOption "filtering out tests already passed by Hydra" // {
            default = true;
          };
        };
        nesting = {
          enable = lib.mkEnableOption "nesting harvested tests under a single check name";
          name = lib.mkOption {
            type = lib.types.str;
            default = "nixos-tests";
            description = "Name of the linkFarm derivation grouping all harvested tests.";
          };
        };
      };

      optionsDefaults = options-schema.lib.getDefaults options;

      # ── Primitives ──────────────────────────────────────────────────────────

      # Returns the nixosTest derivation for a service name, or null if absent.
      matchTest = pkgs: serviceName: pkgs.nixosTests.${serviceName} or null;

      # Returns true if the service uses its default nixpkgs package,
      # meaning the test is already covered by Hydra's standard evaluation.
      isHydraDefault =
        { options, config }:
        serviceName:
        let
          srvOpt = options.services.${serviceName};
          pkg = config.services.${serviceName}.package;
          defaultPkg = srvOpt.package.default;
        in
        srvOpt ? package.default
        && (
          if lib.isDerivation pkg && lib.isDerivation defaultPkg then
            pkg.outPath == defaultPkg.outPath
          else
            pkg == defaultPkg
        );

      # ── Filter pipeline ─────────────────────────────────────────────────────

      # Returns the names of all enabled and visible services from an evaluated NixOS config.
      filterEnabledServices =
        { options, config }:
        lib.filter (
          serviceName:
          let
            srvOpt = options.services.${serviceName};
          in
          srvOpt ? enable && srvOpt.enable.visible or true && config.services.${serviceName}.enable
        ) (lib.attrNames options.services);

      # Returns the pkgs.nixosTests entries for the given service names,
      # excluding services matching the custom and heavy filterFinal.
      filterTests =
        {
          pkgs,
          services,
          filter,
        }:
        let
          filterFinal = filter.custom ++ lib.optionals filter.heavy.enable filter.heavy.list;
          allowedServicesSet = lib.removeAttrs (genAttrsNull services) filterFinal;
        in
        lib.intersectAttrs allowedServicesSet pkgs.nixosTests;

      # ── Internals ────────────────────────────────────────────────────────────

      # Reads the harvester config from an evaluated NixOS configuration,
      # falling back to options defaults if the module is not imported.
      harvesterCfgOf =
        nixosConfiguration: nixosConfiguration.config.services.nixos-test-harvester or optionsDefaults;

      # ── Higher-level ─────────────────────────────────────────────────────────

      # Resolves all relevant nixos tests for a single evaluated NixOS configuration.
      #
      # Steps:
      #   1. Bail out early if the harvester is not enabled in this host's config.
      #   2. Collect all service names where `services.<name>.enable = true` and the option
      #      is not hidden (filterEnabledServices).
      #   3. Optionally drop services that use their default nixpkgs package: those are
      #      already tested by Hydra, so running them again in your CI is redundant
      #      (isHydraDefault).
      #   4. Pass the remaining names to filterTests, which drops heavy/custom filtered
      #      entries and returns the matching pkgs.nixosTests derivations.
      #
      # Always returns a flat { serviceName = drv; } attrset.
      # Nesting is the caller's responsibility.
      resolveTests =
        { nixosConfiguration, pkgs }:
        let
          inherit (nixosConfiguration) config options;
          harvesterCfg = harvesterCfgOf nixosConfiguration;
          enabled = filterEnabledServices { inherit options config; };
        in
        lib.optionalAttrs harvesterCfg.enable (filterTests {
          inherit pkgs;
          filter = harvesterCfg.filter;
          services =
            if harvesterCfg.filter.hydra then
              lib.filter (svc: !isHydraDefault { inherit options config; } svc) enabled
            else
              enabled;
        });

      # Aggregates resolveTests over an attrset of nixosConfigurations, producing
      # a checks-compatible { system = { name = drv; }; } attrset.
      #
      # Steps:
      #   1. For each host, call resolveTests and tag each test as "${testName}-${hostName}"
      #      to avoid name collisions across hosts on the same system.
      #   2. If nesting is enabled for that host, wrap all its tests into a single
      #      linkFarmFromDrvs derivation instead of exposing them individually.
      #   3. Each host emits { system = innerAttrset } or {} (via optionalAttrs).
      #   4. zipAttrsWith collects inner attrsets into per-system lists, then mergeAttrsList
      #      merges each list using binary tree splitting — O(nm log n) vs O(n²m) for a
      #      naive foldl, where n = hosts and m = tests per host.
      harvestAll =
        { hosts, lib }:
        lib.zipAttrsWith (_: lib.mergeAttrsList) (
          lib.mapAttrsToList (
            hostName: nixosConfiguration:
            let
              inherit (nixosConfiguration) pkgs;
              cfg = harvesterCfgOf nixosConfiguration;
              tests = resolveTests { inherit nixosConfiguration pkgs; };
            in
            lib.optionalAttrs (tests != { }) {
              ${pkgs.stdenv.hostPlatform.system} =
                if cfg.nesting.enable then
                  {
                    "${cfg.nesting.name}-${hostName}" = pkgs.linkFarmFromDrvs "${cfg.nesting.name}-${hostName}" (
                      lib.attrValues tests
                    );
                  }
                else
                  lib.mapAttrs' (name: lib.nameValuePair "${name}-${hostName}") tests;
            }
          ) hosts
        );

      # Runs all tests from pkgs.nixosTests with the given filter applied.
      # Does not require a NixOS configuration; useful for running the full test suite.
      runAll =
        {
          pkgs,
          filter ? optionsDefaults.filter,
        }:
        filterTests {
          inherit pkgs filter;
          services = lib.attrNames pkgs.nixosTests;
        };

    in
    {
      nixosModules.default = { ... }: {
        options.services.nixos-test-harvester = options;
      };

      lib = {
        inherit
          matchTest
          isHydraDefault
          filterEnabledServices
          filterTests
          resolveTests
          harvestAll
          runAll
          ;
      };

      # flake-parts module: automatically populates flake.checks from all nixosConfigurations.
      flakeModules.default =
        { config, lib, ... }:
        {
          config.flake.checks = harvestAll {
            hosts = config.flake.nixosConfigurations;
            inherit lib;
          };
        };
    };
}