README.txt

I've got three nodes in an AWS VPC: a jumpbox accessible by external SSH (configuration file: jumpbox.nix), a
k8s master node (configuration file: k8s-master.nix) and a k8s worker node (configuration file: k8s-slave.nix)
in a private subnet (with all traffic allowed between them).

All these .nix files, and the deploy.sh file, are copied to ~/nixos on the jumpbox and then deploy.sh run to set
everything up.

DNS is set up so that these domains resolve to the right machines:
- k8s-master.govuk-k8s.test
- k8s-slave.govuk-k8s.test

The problem I'm having is that the k8s-slave machine doesn't connect to the apiserver.  There are a lot of errors
about permissions:

6099 event.go:240] Server rejected event '&v1.Event{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"ip-10-0-0-25.eu-west-2.compute.internal.15d535d
d1720c0aa", GenerateName:"", Namespace:"default", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Initializers:(*v1.Initializers)(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, InvolvedObject:v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-0-25.eu-west-2.compute.internal", UID:"ip-10-0-0-25.eu-west-2.compute.internal", APIVersion:"", ResourceVersion:"", FieldPath:""}, Reason:"NodeHasSufficientMemory", Message:"Node ip-10-0-0-25.eu-west-2.compute.internal status is now: NodeHasSufficientMemory", Source:v1.EventSource{Component:"kubelet", Host:"ip-10-0-0-25.eu-west-2.compute.internal"}, FirstTimestamp:v1.Time{Time:time.Time{wall:0xbf697c551f7778aa, ext:299577297, loc:(*time.Location)(0x76095a0)}}, LastTimestamp:v1.Time{Time:time.Time{wall:0xbf697c5581612d3a, ext:1794799714, loc:(*time.Location)(0x76095a0)}}, Count:5, Type:"Normal", EventTime:v1.MicroTime{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, Series:(*v1.EventSeries)(nil), Action:"", Related:(*v1.ObjectReference)(nil), ReportingController:"", ReportingInstance:""}': 'events "ip-10-0-0-25.eu-west-2.compute.internal.15d535dd1720c0aa" is forbidden: User "system:node:" cannot patch resource "events" in API group "" in the namespace "default": unknown node for user "system:node:"' (will not retry!)

6099 event.go:240] Server rejected event '&v1.Event{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"ip-10-0-0-25.eu-west-2.compute.internal.15d535dd1720e98d", GenerateName:"", Namespace:"default", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Initializers:(*v1.Initializers)(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, InvolvedObject:v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-0-25.eu-west-2.compute.internal", UID:"ip-10-0-0-25.eu-west-2.compute.internal", APIVersion:"", ResourceVersion:"", FieldPath:""}, Reason:"NodeHasNoDiskPressure", Message:"Node ip-10-0-0-25.eu-west-2.compute.internal status is now: NodeHasNoDiskPressure", Source:v1.EventSource{Component:"kubelet", Host:"ip-10-0-0-25.eu-west-2.compute.internal"}, FirstTimestamp:v1.Time{Time:time.Time{wall:0xbf697c551f77a18d, ext:299587764, loc:(*time.Location)(0x76095a0)}}, LastTimestamp:v1.Time{Time:time.Time{wall:0xbf697c5581615a1f, ext:1794811223, loc:(*time.Location)(0x76095a0)}}, Count:5, Type:"Normal", EventTime:v1.MicroTime{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, Series:(*v1.EventSeries)(nil), Action:"", Related:(*v1.ObjectReference)(nil), ReportingController:"", ReportingInstance:""}': 'events "ip-10-0-0-25.eu-west-2.compute.internal.15d535dd1720e98d" is forbidden: User "system:node:" cannot patch resource "events" in API group "" in the namespace "default": unknown node for user "system:node:"' (will not retry!)

6099 event.go:240] Server rejected event '&v1.Event{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"ip-10-0-0-25.eu-west-2.compute.internal.15d535dd1720f818", GenerateName:"", Namespace:"default", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Initializers:(*v1.Initializers)(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, InvolvedObject:v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-0-25.eu-west-2.compute.internal", UID:"ip-10-0-0-25.eu-west-2.compute.internal", APIVersion:"", ResourceVersion:"", FieldPath:""}, Reason:"NodeHasSufficientPID", Message:"Node ip-10-0-0-25.eu-west-2.compute.internal status is now: NodeHasSufficientPID", Source:v1.EventSource{Component:"kubelet", Host:"ip-10-0-0-25.eu-west-2.compute.internal"}, FirstTimestamp:v1.Time{Time:time.Time{wall:0xbf697c551f77b018, ext:299591487, loc:(*time.Location)(0x76095a0)}}, LastTimestamp:v1.Time{Time:time.Time{wall:0xbf697c5581616c7f, ext:1794815911, loc:(*time.Location)(0x76095a0)}}, Count:5, Type:"Normal", EventTime:v1.MicroTime{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, Series:(*v1.EventSeries)(nil), Action:"", Related:(*v1.ObjectReference)(nil), ReportingController:"", ReportingInstance:""}': 'events "ip-10-0-0-25.eu-west-2.compute.internal.15d535dd1720f818" is forbidden: User "system:node:" cannot patch resource "events" in API group "" in the namespace "default": unknown node for user "system:node:"' (will not retry!)

6099 reflector.go:125] k8s.io/kubernetes/pkg/kubelet/kubelet.go:454: Failed to list *v1.Node: nodes "ip-10-0-0-25.eu-west-2.compute.internal" is forbidden: User "system:node:" cannot list resource "nodes" in API group "" at the cluster scope: unknown node for user "system:node:"

6099 reflector.go:125] k8s.io/kubernetes/pkg/kubelet/kubelet.go:445: Failed to list *v1.Service: services is forbidden: User "system:node:" cannot list resource "services" in API group "" at the cluster scope: unknown node for user "system:node:"

6099 reflector.go:125] k8s.io/kubernetes/pkg/kubelet/config/apiserver.go:47: Failed to list *v1.Pod: pods is forbidden: User "system:node:" cannot list resource "pods" in API group "" at the cluster scope: unknown node for user "system:node:"

And about not being able to identify the node:

6099 kubelet.go:2252] node "ip-10-0-0-25.eu-west-2.compute.internal" not found

Any ideas?

common.nix

{
  imports = [ <nixpkgs/nixos/modules/virtualisation/amazon-image.nix> ];
  ec2.hvm = true;

  # we have security groups
  networking.firewall.enable = false;

  # only keep the last 1GiB of systemd journal
  services.journald.extraConfig = "SystemMaxUse=1G";

  # collect nix store garbage and optimise daily
  nix.gc.automatic = true;
  nix.optimise.automatic = true;
}

deploy.sh

#!/bin/sh

function build_host () {
  host="$1"
  config="$2"
  scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "nixos/common.nix"    "${host}.govuk-k8s.test:/etc/nixos/common.nix"
  scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "nixos/${config}.nix" "${host}.govuk-k8s.test:/etc/nixos/configuration.nix"
  ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "${host}.govuk-k8s.test" nixos-rebuild switch
}

set -ex

build_host k8s-master k8s-master
secret=$(ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no k8s-master.govuk-k8s.test cat /var/lib/kubernetes/secrets/apitoken.secret)

build_host k8s-slave k8s-slave
echo $secret | ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no k8s-slave.govuk-k8s.test nixos-kubernetes-node-join

cp nixos/common.nix  /etc/nixos/common.nix
cp nixos/jumpbox.nix /etc/nixos/configuration.nix
nixos-rebuild switch

jumpbox.nix

{
  imports = [ ./common.nix ];
}

k8s-master.nix

{ pkgs, ... }:

{
  imports = [ ./common.nix ];

  services.kubernetes = {
    masterAddress = "k8s-master.govuk-k8s.test";
    roles = ["master"];
  };

  # necessary for the command-line tools to be in the system PATH; not
  # necessary to start the service running.
  environment.systemPackages = [
    pkgs.kubernetes
  ];
}

k8s-slave.nix

{ pkgs, ... }:

{
  imports = [ ./common.nix ];

  services.kubernetes = {
    masterAddress = "k8s-master.govuk-k8s.test";
    roles = ["node"];
  };

  # necessary for the command-line tools to be in the system PATH; not
  # necessary to start the service running.
  environment.systemPackages = [
    pkgs.kubernetes
  ];
}