01-multi.nix

-- So I want to set up three machine, with two builder nodes having ssh access to the cache to be able
-- to copy closure to that cache machine so that nix-serve can serve up that binary cache so that both
-- builders have a common cache that they can push to and share. This is simulating the setup you may 
-- want if you had an existing CI cluster but wanted to build things on it while having a cluster wide
-- cache.
let
  pub = import ./nix-cache-key.pub;
  sec = import ./nix-cache-key.sec;
  builder = keyname: { config, nodes, pkgs, resources, lib, ... }: {
    deployment.keys.privateKey = {
      text = (lib.traceVal resources.sshKeyPairs.${keyname}.privateKey);
      user = "root";
      group = "root";
      permissions = "0600";
    };
    nix = {
      binaryCaches = ["https://cache.nixos.org/" "http://cache:5000"];
      binaryCachePublicKeys = [pub];
    };
  };
in {
  network.description = "Test Build nodes";
  resources.sshKeyPairs = {
    builder1Key = {};
    builder2Key = {};
  };
  cache = { config, pkgs, resources, lib, ... }:
    {
      users.extraUsers.nix-serve.openssh.authorizedKeys.keys = [
        resources.sshKeyPairs.builder1Key.publicKey
        resources.sshKeyPairs.builder2Key.publicKey
      ];
      deployment.keys.signing-key = {
        text = sec;
        user = "nix-serve";
      };
      services.nix-serve = {
        enable = true;
        secretKeyFile = "/run/keys/signing-key";
      };
      networking.firewall.allowedTCPPorts = [22 5000];
    };
  builder1 = builder "builder1Key";
  builder2 = builder "builder2Key";
}

02-multi-vbox.nix

let
  machine =
    { config, pkgs, ... }:
    { deployment.targetEnv = "virtualbox";
      deployment.virtualbox.headless = true;
      deployment.virtualbox.memorySize = 1024; # megabytes
      deployment.virtualbox.vcpu = 2; # number of cpus
    };
in {
  cache = machine;
  builder1 = machine;
  builder2 = machine;
}

03-logs

bkolera at bkolera-qfpl in ~/src/github/benkolera/playground/nix/nixops/vbox-manual (master●)
$ nixops deploy -d multi --force-reboot
trace:
trace:
building all machine configurations...
trace: -----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
QyNTUxOQAAACDzXxhfpRYM3AgZddSbHG2sNybQNZBOeeea8VoQZyeSsQAAAKB+/Rf5fv0X
+QAAAAtzc2gtZWQyNTUxOQAAACDzXxhfpRYM3AgZddSbHG2sNybQNZBOeeea8VoQZyeSsQ
AAAEBgEan/rx0TjUHAZ7Zb2quS6SxWwfBBKLWfwwwe8a2FRfNfGF+lFgzcCBl11Jscbaw3
JtA1kE5555rxWhBnJ5KxAAAAGU5peE9wcyBhdXRvLWdlbmVyYXRlZCBrZXkBAgME
-----END OPENSSH PRIVATE KEY-----

trace: -----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
QyNTUxOQAAACAva2W+OmRo0FFnyYjCHIOJLYva23f/2MtC6M48Qi8uVAAAAKAa8bWYGvG1
mAAAAAtzc2gtZWQyNTUxOQAAACAva2W+OmRo0FFnyYjCHIOJLYva23f/2MtC6M48Qi8uVA
AAAEDFfTE7FxwClQjfQbA6aqDDcT17LIiEeX0vvZnTtModES9rZb46ZGjQUWfJiMIcg4kt
i9rbd//Yy0LozjxCLy5UAAAAGU5peE9wcyBhdXRvLWdlbmVyYXRlZCBrZXkBAgME
-----END OPENSSH PRIVATE KEY-----

cache......> copying closure...
builder2...> copying closure...
builder1...> copying closure...
multi> closures copied successfully
builder2...> uploading key ‘privateKey’...
cache......> uploading key ‘signing-key’...
builder1...> uploading key ‘privateKey’...
builder1...> updating GRUB 2 menu...
cache......> updating GRUB 2 menu...
builder2...> updating GRUB 2 menu...
builder1...> rebooting...
cache......> rebooting...
builder2...> rebooting...
cache......> waiting for the machine to finish rebooting...
builder2...> waiting for the machine to finish rebooting...
builder1...> waiting for the machine to finish rebooting...
cache......> [down]
builder2...> .
builder1...> .
cache......> .
builder2...> [down]
builder1...> [down]
builder1...> .[up]
builder1...> uploading key ‘privateKey’...
cache......> .
builder2...> .
builder1...> activation finished successfully
cache......> .
builder2...> .
cache......> .
builder2...> .
cache......> .
builder2...> .
cache......> .
builder2...> .[up]
builder2...> uploading key ‘privateKey’...
cache......> .[up]
cache......> uploading key ‘signing-key’...
builder2...> activation finished successfully
cache......> activation finished successfully
multi> deployment finished successfully

bkolera at bkolera-qfpl in ~/src/github/benkolera/playground/nix/nixops/vbox-manual (master●)
$ nixops ssh -d multi builder1

Last login: Thu Oct  4 06:08:25 2018 from 192.168.56.1

[root@builder1:~]# cat /run/keys/privateKey

[root@builder1:~]#

04-Notes

It looks like the config is collected for the deployment.keys before resources is actually populated. The trace 
line shows two lines for each machine and the first time it is blank and the second time it has a private key.

But the file on disk is empty. It looks like this is a common-enough problem in nixops with not enough laziness in
collecting thing up for the deployment. See: https://github.com/NixOS/nixops/commit/140b2593cca1df82b3800ae147d5b0e830c54daa

Is there a better way to do this other than to actually have keypairs in the git repo next to the nix files or 
adding laziness to deployment.keys.name?.text?