third try is a charm

hcloud_nixos.go

package main

import (
	"context"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"strings"
	"time"

	"github.com/hetznercloud/hcloud-go/hcloud"
	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/agent"
)

var (
	hcloudToken    = flag.String("token", "", "Token for Hetzner Cloud")
	hcloudSSH      = flag.Int("ssh", -1, "SSH Key IDs")
	hcloudServerID = flag.Int("server", -1, "id of server")
)

func main() {
	flag.Parse()
	if *hcloudToken == "" {
		token := os.Getenv("TF_VAR_hcloud_token")
		hcloudToken = &token
	}

	client := hcloud.NewClient(hcloud.WithToken(*hcloudToken))

	ctx := context.Background()

	server, _, err := client.Server.GetByID(ctx, *hcloudServerID)
	if err != nil {
		log.Fatalf("error retrieving server: %s\n", err)
	}

	if server != nil {
		fmt.Printf("server %d is called %q\n", server.ID, server.Name)
	} else {
		fmt.Printf("server %d not found\n", *hcloudServerID)
	}

	key, _, err := client.SSHKey.GetByID(ctx, *hcloudSSH)
	fail("Failed finding SSH Key", err)

	for server.Status == hcloud.ServerStatusInitializing {
		log.Println(server.Status)
		time.Sleep(time.Second)
		server, _, err = client.Server.GetByID(ctx, *hcloudServerID)
	}

	client.Server.EnableRescue(ctx, server, hcloud.ServerEnableRescueOpts{
		SSHKeys: []*hcloud.SSHKey{key},
		Type:    hcloud.ServerRescueTypeLinux64,
	})

	reboot(ctx, client, server)

	sshConfig := &ssh.ClientConfig{
		User:            "root",
		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
		Auth:            []ssh.AuthMethod{dialSSHAgent()},
	}

	log.Println("Connecting to SSH on", server.PublicNet.IPv4.IP.String())

	var sshClient *ssh.Client
	for n := 0; n < 20; n++ {
		sshClient, err = ssh.Dial("tcp", fmt.Sprintf("%s:22", server.PublicNet.IPv4.IP.String()), sshConfig)
		if err == nil {
			log.Println("connected")
			doneChan := make(chan bool, 1)
			withSSHSession(sshClient, false, func(sshSession *ssh.Session) {
				output, err := sshSession.CombinedOutput(`ls`)
				if err == nil {
					if strings.Contains(string(output), "hwcheck-logs") {
						log.Println("We made it into rescue mode")
						doneChan <- true
						return
					} else {
						log.Println("This isn't rescue mode, trying again")
						client.Server.EnableRescue(ctx, server, hcloud.ServerEnableRescueOpts{
							SSHKeys: []*hcloud.SSHKey{key},
							Type:    hcloud.ServerRescueTypeLinux64,
						})

						reboot(ctx, client, server)
					}
				}
				doneChan <- false
			})
			if <-doneChan {
				break
			}
		}
		log.Println("retrying connection...", err)
		time.Sleep((time.Second * 1) * time.Duration(n))
	}

	if sshClient == nil {
		fail("Couldn't connect", err)
	}

	defer sshClient.Close()

	for _, rawCmd := range []string{
		`mkdir -p -m 0755 /nix && chown root /nix`,
		`mkdir -p /etc/nix`,
		`echo "build-users-group =" > /etc/nix/nix.conf`,
		`curl https://nixos.org/nix/install | sh`,
		`echo ". $HOME/.nix-profile/etc/profile.d/nix.sh" > $HOME/.bashrc`,
		`nix-env -iE "_: with import <nixpkgs/nixos> { configuration = {}; }; with config.system.build; [ nixos-generate-config nixos-install ]"`,
		`echo -e "d\nn\np\n\n\n\nw" | fdisk /dev/sda`,
		`mkfs.ext4 -F /dev/sda1`,
		`mount /dev/sda1 /mnt`,
		`nixos-generate-config --root /mnt/`,
		`cat > /mnt/etc/nixos/configuration.nix <<EOF
{config, pkgs, ...}:
{
  imports = [ ./hardware-configuration.nix ];
  boot.loader.grub.enable = true;
  boot.loader.grub.version = 2;
	boot.loader.grub.device = "/dev/sda"; # or "nodev" for efi only
  system.stateVersion = "19.03"; # Did you read the comment?
  services.openssh.enable = true;
  services.openssh.permitRootLogin = "yes";
  users.users.root.openssh.authorizedKeys.keys = [ "` + key.PublicKey + `" ];
}
EOF
`,
		// `nix-env -p /nix/var/nix/profiles/system -f '<nixpkgs/nixos>' -I nixos-config=/mnt/etc/nixos/configuration.nix -iA system`,
		`nixos-install --no-root-passwd`,
	} {
		func(cmd string) {
			withSSHSession(sshClient, true, func(sshSession *ssh.Session) {
				log.Println(cmd)
				fail(cmd, sshSession.Run(cmd))
			})
		}(rawCmd)
	}

	disableRescue(ctx, client, server)
	reboot(ctx, client, server)
}

func withSSHSession(sshClient *ssh.Client, pipes bool, fun func(*ssh.Session)) {
	sshSession, err := sshClient.NewSession()
	fail("Couldn't create SSH session", err)
	defer sshSession.Close()

	modes := ssh.TerminalModes{
		ssh.ECHO:          0,         // disable echoing
		ssh.TTY_OP_ISPEED: 14400 * 2, // input speed = 14.4kbaud
		ssh.TTY_OP_OSPEED: 14400 * 2, // output speed = 14.4kbaud
	}

	if err := sshSession.RequestPty("xterm", 160, 40, modes); err != nil {
		fail("request for pseudo terminal failed", err)
	}

	if pipes {
		setupSSHPipes(sshSession)
	}

	fun(sshSession)
}

func reboot(ctx context.Context, client *hcloud.Client, server *hcloud.Server) {
	action, _, err := client.Server.Reboot(ctx, server)
	fail("Couldn't reboot", err)
	waitForServerAction(ctx, client, action, server)
}

func disableRescue(ctx context.Context, client *hcloud.Client, server *hcloud.Server) {
	action, _, err := client.Server.DisableRescue(ctx, server)
	fail("Couldn't disable rescue", err)
	waitForServerAction(ctx, client, action, server)
}

func fail(detail string, err error) {
	if err != nil {
		log.Fatalln(detail, err)
	}
}

func waitForServerAction(ctx context.Context, client *hcloud.Client, action *hcloud.Action, server *hcloud.Server) error {
	log.Printf("[INFO] server (%d) waiting for %q action to complete...", server.ID, action.Command)
	_, errCh := client.Action.WatchProgress(ctx, action)
	if err := <-errCh; err != nil {
		return err
	}
	log.Printf("[INFO] server (%d) %q action succeeded", server.ID, action.Command)
	return nil
}

func dialSSHAgent() ssh.AuthMethod {
	sshAgent, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
	if err != nil {
		fail("Couldn't find SSH_AUTH_SOCK", err)
	}

	return ssh.PublicKeysCallback(agent.NewClient(sshAgent).Signers)
}

func setupSSHPipes(sshSession *ssh.Session) {
	stdin, err := sshSession.StdinPipe()
	if err != nil {
		fail("Unable to setup stdin for session", err)
	}
	go io.Copy(stdin, os.Stdin)

	stdout, err := sshSession.StdoutPipe()
	if err != nil {
		fail("Unable to setup stdout for session", err)
	}
	go io.Copy(os.Stdout, stdout)

	stderr, err := sshSession.StderrPipe()
	if err != nil {
		fail("Unable to setup stderr for session", err)
	}
	go io.Copy(os.Stderr, stderr)
}

server.tf

variable hcloud_token {
  type = "string"
}

provider "hcloud" {
  token = "${var.hcloud_token}"
}

resource "hcloud_ssh_key" "default" {
  name = "manveru@kappa"
  public_key = "${file("~/.ssh/id_ed25519.pub")}"
}

resource "hcloud_server" "axe" {
  name = "axe"
  server_type = "cx11"
  location = "nbg1"
  image = "debian-9" # we wipe this immediately
  ssh_keys = ["${hcloud_ssh_key.default.id}"]

  provisioner "local-exec" {
    command = "go run hcloud_nixos.go -token '${var.hcloud_token}' -server ${self.id} -ssh ${hcloud_ssh_key.default.id}"
  }
}