kubernetes.nix

{ config, pkgs, ... }:

{
  # Kubernetes configuration
  # Insecure, for local development only, totally unsuitable for production
  services.kubernetes = {
    roles = ["master" "node"];
    
    # Without explicitly defined keys things will break after reboot,
    # as by default keys will be generated in /var/run/kubernetes.
    # Note, for simplicity and laziness sake, a single keypair is used
    # for CA, server and client keys - which is totally insecure.
    caFile = "/etc/nixos/private/k8s.crt";
    kubeconfig = {
      keyFile = "/etc/nixos/private/k8s.key";
      certFile = "/etc/nixos/private/k8s.crt";
    };
    apiserver = {
      authorizationMode = [ "AlwaysAllow" ];
      tlsCertFile = "/etc/nixos/private/k8s.crt";
      tlsKeyFile = "/etc/nixos/private/k8s.key";
    };
    controllerManager = {
      rootCaFile = "/etc/nixos/private/k8s.crt";
      serviceAccountKeyFile = "/etc/nixos/private/k8s.key";
    };
    kubelet = {
      tlsKeyFile = "/etc/nixos/private/k8s.key";
      tlsCertFile = "/etc/nixos/private/k8s.crt";
      extraOpts = "--fail-swap-on=false --eviction-hard=memory.available<128Mi,nodefs.available<512Mi,imagefs.available<512Mi,nodefs.inodesFree<5%";
    };
  };
}

openssl.conf

# NOTE: Replace "myhostname" in alt_names section with an appropriate value.

[ req ]
default_bits = 2048
prompt = no
default_md = sha256
req_extensions = req_ext
distinguished_name = dn

[ dn ]
CN = 127.0.0.1

[ req_ext ]
subjectAltName = @alt_names

[ alt_names ]
DNS.1 = kubernetes
DNS.2 = kubernetes.default
DNS.3 = kubernetes.default.svc
DNS.4 = kubernetes.default.svc.cluster
DNS.5 = kubernetes.default.svc.cluster.local
DNS.6 = myhostname
DNS.7 = localhost
IP.1 = 127.0.0.1
IP.2 = 10.0.0.1

[ v3_ext ]
authorityKeyIdentifier=keyid,issuer:always
basicConstraints=CA:TRUE
keyUsage=keyEncipherment,dataEncipherment,keyCertSign,digitalSignature
extendedKeyUsage=serverAuth,clientAuth
subjectAltName=@alt_names

openssl.sh

#!/bin/sh
[ -e k8s.key ] || openssl genrsa -out k8s.key 2048
openssl req -x509 -new -nodes -key k8s.key -subj "/CN=127.0.0.1" -days 3650 -out k8s.crt -config openssl.conf -extensions v3_ext
chown kubernetes:kubernetes k8s.key k8s.crt
chmod 0640 k8s.key
chmod 0644 k8s.crt

Note: if networking.firewall.enabled then one may need something like networking.firewall.extraCommands = "ip46tables -A nixos-fw -i cbr0 -j ACCEPT"; (I haven't thought about a good rule) or pods will not be able to reach the apiserver.

Symptoms: on NixOS (specifically) kubernetes fails to start after reboot. kube-dns pod gets stuck with 2/3 CrashLoopBackOff, and there are lots of "Unable to authenticate the request due to an error: [invalid bearer token, [invalid bearer token, crypto/rsa: verification error]]" from kube-apiserver.

Also this "fixes" (by applying very permissive AlwaysAllow policy, so totally not secure) inability to fetch logs. Without this:

$ kubectl -n kube-system logs kube-dns-5d585466b5-jtq99 -p kubedns
Error from server (Forbidden): Forbidden (user=system:anonymous, verb=get, resource=nodes, subresource=proxy) ( pods/log kube-dns-5d585466b5-jtq99)