second approach for terraform&nixos&hcloud

hcloud_stage_1.sh

#!/usr/bin/env bash

set -ex

apt-get install -y squashfs-tools git
rm -rf nixos-in-place
if [ ! -d nixos-in-place ]; then
    git clone https://github.com/manveru/nixos-in-place.git
fi
cd nixos-in-place
git checkout 329cee1bd6db5c4463be77908fb44a8f88254e17

echo "y\nn" | ./install

echo "exited with $?"

hnix_nixos_in_place.cr

require "http/client"
require "json"

class HetznerServer
  class Response
    JSON.mapping(action: Action)
  end

  class Action
    JSON.mapping(
      id: Int32,
      command: String,
      status: String,
      progress: Int32,
    )
  end

  def initialize(@server_id : Int32, @server_ip : String, @ssh_key_id : Int32)
    @client = HTTP::Client.new(URI.parse("https://api.hetzner.cloud"))
    @client.before_request do |request|
      request.headers["Content-Type"] = "application/json"
      request.headers["Authorization"] = "Bearer #{ENV["TF_VAR_hcloud_token"]}"
    end
  end

  def enable_rescue
    post "enable_rescue", {"type": "linux64", "ssh_keys": [@ssh_key_id]}
  end

  def disable_rescue
    post "disable_rescue"
  end

  def attach_iso(iso_name : String)
    post "attach_iso", {"iso": iso_name}
  end

  def detach_iso
    post "detach_iso"
  end

  def reboot
    post "reboot"
  end

  def post(action : String)
    puts "POST #{action}"
    response = @client.post("/v1/servers/#{@server_id}/actions/#{action}")
    puts response.body
    wait_for_success Response.from_json(response.body).action.id
  rescue ex : JSON::MappingError
    puts response.body if response
    raise ex
  end

  def post(action : String, body)
    puts "POST #{action} #{body.to_json}"
    response = @client.post("/v1/servers/#{@server_id}/actions/#{action}", body: body.to_json)
    puts response.body
    wait_for_success Response.from_json(response.body).action.id
  rescue ex : JSON::MappingError
    puts response.body if response
    raise ex
  end

  def action_status(action_id)
    puts "GET #{action_id}"
    response = @client.get("/v1/servers/#{@server_id}/actions/#{action_id}")
    Response.from_json(response.body).action
  rescue ex : JSON::MappingError
    puts response.body if response
    raise ex
  end

  def get(action)
    puts "GET #{action}"
    response = @client.get("/v1/servers/#{@server_id}/actions/#{action}")
    wait_for_success Response.from_json(response.body).action.id
  rescue ex : JSON::MappingError
    puts response.body if response
    raise ex
  end

  def wait_for_success(action_id)
    current = action_status(action_id)

    puts "Waiting for #{current.command}##{current.id}..."

    10.times do
      if current
        case current.status
        when "success"
          puts " done"
          return current
        when "running"
          print "."
        else
          puts " fail"
          raise "Unknown status returned: #{current.status}"
        end
      end

      sleep 1

      current = action_status(action_id)
    end

    raise "Waiting for success timed out"
  end

  def exec(exe : String, args : Array(String))
    puts "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv"
    puts "#{exe} #{args.join(" ")}"
    # error = IO::Memory.new
    # stdout = IO::Memory.new
    status = Process.run(exe, args: args, output: STDOUT, error: STDERR)
    # puts "stdout: #{stdout}"
    # puts "stderr: #{error}"
    raise "FAIL: #{exe} #{args.join(" ")}" unless status.success?
    puts "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
    status
  end

  def retrying(n)
    tries = 0

    loop do
      begin
        tries += 1
        return yield
      rescue ex : Exception
        if tries > n
          puts "giving up after #{n} attempts"
          raise ex
        end
        puts ex
        sleep 1
      end
    end
  end

  def ssh(args : Array(String))
    retrying 10 do
      exec("ssh", ["-oStrictHostKeyChecking=no", "-oUserKnownHostsFile=/dev/null",
                   "root@#{@server_ip}"] + args)
    end
  end

  def scp(file, dest)
    retrying 10 do
      exec("scp", ["-oStrictHostKeyChecking=no", "-oUserKnownHostsFile=/dev/null",
                   file, "root@#{@server_ip}:#{dest}"])
    end
  end
end

server_ip = ARGV[0]
server_id = ARGV[1].to_i32
ssh_key_id = ARGV[2].to_i32

server = HetznerServer.new(server_id, server_ip, ssh_key_id)

# server.disable_rescue
# server.reboot
server.scp "hcloud_stage_1.sh", "/hcloud_stage_1.sh"
server.ssh ["bash", "/hcloud_stage_1.sh"]
server.enable_rescue
server.reboot
server.scp "hcloud_stage_2.sh", "/hcloud_stage_2.sh"
server.ssh ["bash", "/hcloud_stage_2.sh"]
server.disable_rescue
server.reboot

puts "All done"

hnix_stage_2.sh

#!/usr/bin/env bash

set -ex

(mount | grep /mnt) || mount /dev/sda1 /mnt/

cd /mnt

shopt -s extglob
if [ -d nixos ]; then
    rm -rf --one-file-system !(nixos)
    rm -rf nixos/old-root/
    mv nixos/* .
    rmdir nixos
fi

(mount | grep /mnt/proc) || mount -t proc  proc      /mnt/proc/
(mount | grep /mnt/sys)  || mount -t sysfs sys       /mnt/sys/
(mount | grep /mnt/dev)  || mount -o bind  /dev      /mnt/dev/

cat > /mnt/etc/nixos/nixos-in-place.nix <<EOF
{ config, pkgs, ... }:
{
  boot.kernelParams = ["boot.shell_on_fail"];
  boot.loader.grub.device = "/dev/sda";
  boot.loader.grub.storePath = "/nix/store";
  boot.initrd.supportedFilesystems = [ "ext4" ];
  fileSystems = {
    "/" = {
      device = "/dev/sda1";
      fsType = "ext4";
    };
  };
  users.extraUsers.root.password = "nixos";
  services.openssh.enable = true;
  users.extraUsers.root.openssh.authorizedKeys.keys = [
    "$(head -1 /root/.ssh/authorized_keys)"
  ];
}
EOF

# Yes, that's hacky, please tell me a better way :)

cat <<EOF | chroot . "$(echo nix/store/*-bash-*/bin/bash | cut -d' ' -f1)"
set -ex

export PATH="\$(echo -n /nix/store/*-system-path/bin):/usr/bin"
nixos-rebuild switch
grub-install /dev/sda

exit # exit chroot

apt-get install -y zerofree
zerofree -v /dev/sda1
EOF