Distributed nix builds

distributed-master.nix

{ config, pkgs, ... }: {
  services.nfs.server.enable = true;
  services.nfs.server.exports = ''
    /exports/nix 192.168.1.0/255.255.255.0(rw,no_root_squash,no_subtree_check,hide,fsid=0)
    /exports/nix/store 192.168.1.0/255.255.255.0(rw,no_root_squash,no_subtree_check,fsid=1)
  '';
  services.nfs.server.createMountPoints = true;

  programs.ssh.extraConfig = ''
    Port 64123
    Host *
      StrictHostKeyChecking no
  '';
  services.openssh.ports = [ 22 64123 ];

  boot.postBootCommands = ''
    mkdir -m 755 -p /exports/nix
    mkdir -m 1777 -p /exports/nix/store
    mount --bind /nix /exports/nix
    mount --bind /nix/store /exports/nix/store
    mount -o remount,rw /exports/nix/store
    chgrp 30000 /exports/nix/store
  '';
}

distributed-slave.nix

{ config, pkgs, ... }:
  with pkgs.lib;
let builder = config.systemd.containers.builder;
in {
  fileSystems = [{
    mountPoint = "/run/nix/remote_nix";
    device = "server:/";
    fsType = "nfs";
    options = "vers=4,retry=1000";
  }];

  boot.initrd.kernelModules = [ "fuse" ];

  systemd.containers = {
    builder =
      let masterConfig = config; in {
      config = { config, pkgs, ... }:  {
        services.openssh.enable = true;
        services.openssh.ports = [ 64123 ];
        users.extraUsers.root.openssh.authorizedKeys.keys =
          masterConfig.users.extraUsers.root.openssh.authorizedKeys.keys;
      };
    };
  };

  systemd.services.mergestore = {
    description = "merges store from master and container";
    wantedBy = [ "multi-user.target" ];
    unitConfig.RequiresMountsFor = [ "/run/nix/remote_nix/store" ];
    preStart = "mkdir -p /run/nix/merged_stores";
    serviceConfig.ExecStart =
        "${pkgs."unionfs-fuse"}/bin/unionfs -o allow_other,cow,nonempty /run/nix/remote_nix/store=RW:/nix/store=RO /run/nix/merged_stores";
    serviceConfig.Type = "forking";
    serviceConfig.RestartSec = "1min";
    serviceConfig.Restart = "always";
  };

  systemd.services."container-builder".after = [ "mergestore.service" ];
  systemd.services."container-builder".serviceConfig.ExecStart = mkOverride 50
    "${config.systemd.package}/bin/systemd-nspawn -M builder -D ${builder.root} --bind=/run/nix/remote_nix:/nix --bind-ro=/run/nix/merged_stores:/nix/store ${builder.path}/init";
  systemd.services."container-builder".serviceConfig.Restart = "always";
  systemd.services."container-builder".serviceConfig.RestartSec = "1min";
}

shared-store-test.nix

{ pkgs, ... }:
  with pkgs;
  with pkgs.lib;
let
  # Trivial Nix expression to build remotely.
  expr = config: nr: pkgs.writeText "expr.nix"
    ''
      let utils = builtins.storePath ${config.system.build.extraUtils}; in
      derivation {
        name = "hello-${toString nr}";
        system = "i686-linux";
        PATH = "''${utils}/bin";
        builder = "''${utils}/bin/sh";
        args = [ "-c" "echo Hello; mkdir $out; cat /proc/sys/kernel/hostname > $out/host; sleep 3" ];
      }
    '';

  client =
    { config, pkgs, nodes, ... }: {
      require = [ ./distributed-slave.nix ];
      users.extraUsers.root.openssh.authorizedKeys.keys = [ (builtins.readFile ./id_rsa.pub) ];
    };

in {
  nodes = {
    client1 = client;
    #client2 = client;
    server = { config, pkgs, ... }: {
      require = [ ./distributed-master.nix ];

      virtualisation.writableStore = true;
      virtualisation.pathsInNixDB = [ config.system.build.extraUtils ];

      nix.distributedBuilds = true;
      nix.buildMachines = [
        { hostName = "client1";
          sshKey = "/root/.ssh/id_rsa";
          sshUser = "root";
          system = "i686-linux";
          maxJobs = 1;
        }
      ];
    };
  };

  testScript = { nodes }: ''
    startAll;

    $client1->waitForUnit("container-builder.service");
    $client1->waitUntilSucceeds("netstat -nlp | grep sshd");

    $server->succeed("mkdir -m 700 /root/.ssh");
    $server->succeed("cp ${./id_rsa} /root/.ssh/id_rsa && chmod 400 /root/.ssh/id_rsa");

    my $out = $server->succeed("nix-build -j0 ${expr nodes.server.config 1}");
    $server->succeed("test -e $out");
  '';
}