Passing secrets to NixOS containers without exposing them world-readable

module.nix

{ pkgs, lib, config, ... }:

let
  inherit (lib) mkOption types;

  secretsModule = {
    options.path = mkOption {
      type = types.path;
      example = lib.literalExample "pkgs.hello";
      description = ''
        The store path to expose to the container, accessible by the given
        <option>users</option> and <option>groups</option>.
      '';
    };
    options.users = mkOption {
      type = types.listOf types.str;
      default = [];
      description = ''
        List of users with access to the given <option>path</option>.
      '';
    };
    options.groups = mkOption {
      type = types.listOf types.str;
      default = [];
      description = ''
        List of groups with access to the given <option>path</option>.
      '';
    };
  };

  containerModule = { name, config, ... }: {
    options.secrets = mkOption {
      type = types.attrsOf (types.submodule secretsModule);
      default = {};
      example.foo = { path = "/some/path"; users = [ "nextcloud" ]; };
      description = ''
        A mapping between secret names and their corresponding store paths,
        which are made available to the container via a
        <literal>secrets</literal> NixOS module argument.
      '';
    };

    config = {
      bindMounts."/run/container-secrets-host" = {
        hostPath = "/run/container-secrets/${name}";
        isReadOnly = true;
      };

      config = {
        _module.args.secrets = let
          mkPath = sname: lib.const "/run/container-secrets/${sname}/path";
        in lib.mapAttrs mkPath config.secrets;

        systemd.services = lib.mapAttrs' (sname: scfg: {
          name = "secret-dir-${sname}";
          value = {
            description = "Create Directory for Secret ${sname}";

            script = let
              userPerms = map (u: "u:${u}:x") scfg.users;
              groupPerms = map (g: "g:${g}:x") scfg.groups;
              acl = lib.concatStringsSep "," (userPerms ++ groupPerms);
            in ''
              sdir=/run/container-secrets/${lib.escapeShellArg sname}
              mkdir -p "$sdir"
              ${pkgs.acl}/bin/setfacl -b -m ${lib.escapeShellArg acl} "$sdir"
            '';

            unitConfig.DefaultDependencies = false;
            serviceConfig.Type = "oneshot";
            serviceConfig.UMask = "0077";
            serviceConfig.RuntimeDirectory = "container-secrets";
            serviceConfig.RuntimeDirectoryPreserve = true;
          };
        }) config.secrets;

        systemd.mounts = lib.mapAttrsToList (sname: scfg: {
          description = "Secrets for ${sname}";
          requiredBy = [ "local-fs.target" ];
          requires = [ "secret-dir-${sname}.service" ];
          before = [ "local-fs.target" ];
          after = [ "secret-dir-${sname}.service" ];

          what = "/run/container-secrets-host/${sname}";
          where = "/run/container-secrets/${sname}/path";
          type = "none";
          options = "bind,ro";
        }) config.secrets;
      };
    };
  };

in {
  options.containers = mkOption {
    type = types.attrsOf (types.submodule containerModule);
  };

  config.systemd.services = lib.mapAttrs' (name: cfg: {
    name = "container@${name}";
    value = {
      confinement.enable = true;
      confinement.mode = "chroot-only";
      confinement.packages = [ config.containers.${name}.path ];

      serviceConfig.BindPaths = [
        # This is for communicating machine state back to the host via DBus.
        "/run:/run:rbind"
        # Sysfs needs to be read-write because nspawn creates cgroups.
        "/sys:/sys:rbind"
      ] ++ lib.mapAttrsToList (_: lib.getAttr "hostPath") cfg.bindMounts;
      serviceConfig.BindReadOnlyPaths = [
        "/etc/resolv.conf" "/etc/machine-id" "/etc/os-release"
        # Both of these are later mounted again in the container, so we only
        # need them for startup.
        "/dev:/dev:rbind" "/proc:/proc:rbind"
        # The container module tries to bind-mount these, so stub them out.
        "/dev/null:/nix/var/nix/db"
        "/dev/null:/nix/var/nix/daemon-socket"
      ] ++ lib.mapAttrsToList (sname: scfg:
        "${scfg.path}:/run/container-secrets/${name}/${sname}"
      ) cfg.secrets;

      serviceConfig.PrivateTmp = true;
      serviceConfig.StateDirectory = "containers/${name}";
      serviceConfig.RuntimeDirectory = [ "container-secrets/${name}" ];
      serviceConfig.RuntimeDirectoryMode = "0700";
    };
  }) config.containers;
}

test.nix

import <nixpkgs/nixos/tests/make-test-python.nix> {
  name = "container-confinement";

  nodes.machine = { config, pkgs, ... }: {
    imports = [ ./module.nix ];

    users.users.alice.isNormalUser = true;

    containers.foo.autoStart = true;

    containers.foo.secrets.docroot = {
      users = [ "nginx" ];
      path = pkgs.writeTextFile {
        name = "secret-docroot";
        destination = "/index.html";
        text = ''
          <!DOCTYPE html>
          cOnfiDentiAlStuFF
        '';
      };
    };

    containers.foo.config = { secrets, ... }: {
      users.users.bar = {
        description = "User Without Access";
        isNormalUser = true;
      };
      services.nginx.enable = true;
      services.nginx.virtualHosts."localhost".locations."/" = {
        root = secrets.docroot;
      };
      # This is needed so we get output from failures during container startup.
      services.journald.extraConfig = ''
        ForwardToConsole=yes
        MaxLevelConsole=debug
      '';
    };

    containers.unrelated.autoStart = true;
    containers.unrelated.bindMounts."/alice".hostPath = "/home/alice";
    containers.unrelated.config = {};
  };

  testScript = ''
    from shlex import quote

    machine.wait_for_unit('multi-user.target')

    with subtest('cannot access files on the host'):
      machine.fail(
        'nixos-container run foo -- ls -l /nix/store | grep -q secret-docroot',
      )
      machine.succeed(
        'nixos-container run foo -- '
        'getfacl /run/container-secrets/docroot >&2'
      );
      machine.fail(
        'nixos-container run foo -- '
        'su -c "cat /run/container-secrets/docroot/path/index.html" bar >&2'
      );
      machine.fail(
        'nixos-container run foo -- '
        'su -c "cat /run/container-secrets-host/docroot/index.html" bar >&2'
      )

    with subtest('secret cannot be found in files inside the container'):
      machine.succeed(
        'curl http://localhost/index.html | grep -q cOnfiDentiAlStuFF'
      )
      # The "[f]" here is needed so we avoid grepping for the grep command
      # itself in the transient systemd unit.
      cmd = 'out="$(grep -sr "cOn[f]iDentiAlStuFF" /nix /var /run || :)";' \
            ' if [ -n "$out" ]; then echo "$out"; exit 1; fi'
      machine.succeed(f"nixos-container run foo -- su -c {quote(cmd)} bar")

    with subtest('bind-mount is reachable in container without secrets'):
      machine.succeed('echo -n hello world > /home/alice/somefile.txt')
      data = machine.succeed(
        'nixos-container run unrelated cat /alice/somefile.txt'
      )
      assert data == 'hello world', f'{data!r} != "hello world"'
  '';
}

Just fixed the VM test to run with current nixpkgs versions and here is the actual error:

Failed to mount /usr/lib/os-release (type n/a) on /var/lib/nixos-containers/foo/run/host/os-release (MS_BIND ""): No such file or directory

Currently bisecting the issue...

Just fixed the VM test to run with current nixpkgs versions and here is the actual error:

Failed to mount /usr/lib/os-release (type n/a) on /var/lib/nixos-containers/foo/run/host/os-release (MS_BIND ""): No such file or directory

Currently bisecting the issue...

Interesting I did not see that error when looking through logs.

@Saturn745: Fixed. The issue got introduced in the upgrade to systemd v246 and we essentially just need yet another file that we need to pass to the container 😒

@Saturn745: Fixed. The issue got introduced in the upgrade to systemd v246 and we essentially just need yet another file that we need to pass to the container 😒

I will try it when I get back to the computer. I greatly appreciate you taking your time to look into the issue 😃. I have been doing some weird stuff to pass secrets to containers but this module makes things so much easier.

@Saturn745: Fixed. The issue got introduced in the upgrade to systemd v246 and we essentially just need yet another file that we need to pass to the container 😒

Just tried this and while it does work it also breaks all of my other containers which make use bind mounts (for other purposes), saying that the file/directory does not exist when the container is trying to boot.

Failed to stat /some/other/bind/mount: No such file or directory

@Saturn745: The bindMounts option was broken anyway because with this module the nspawned service is confined and doesn't have full access to files on the host, so it was necessary to expose these paths to the service. In other words: The issue should be fixed.