Node Init - Kubeadm

config.go

package nodeinit

import "strings"

// https://coredns.io/plugins/kubernetes/
type CoreDNSOptions struct {
	// the URL for a remote k8s API endpoint
	KubernetesEndpoint string
	// exposes the k8s namespaces listed
	Namespaces string
}

// https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/
type KubeletOptions struct {
	// The provider for cloud services. Specify empty string for running with no cloud provider.
	CloudProvider string
	// Resolver configuration file used as the basis for the container DNS resolution configuration. (default "/etc/resolv.conf")
	ResolvConf string
	// Comma-separated list of DNS server IP address.
	ClusterDNS string
}

type CmdOptions struct {
	ShowVersionAndExit bool

	// DnsHost       bool
	NodeGroup string
	NodeRoles string

	Kubelet KubeletOptions
	CoreDNS CoreDNSOptions
}

func (o *CmdOptions) Roles() []string {
	return strings.Split(o.NodeRoles, ",")
}

gzip.go

// pkg/nodeinit/gzip.go
package nodeinit

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
)

func ExtractTarGz(gzipPath string) error {
	rawData, err := ioutil.ReadFile(gzipPath)
	if err != nil {
		return fmt.Errorf("failed reading file=%s, err=%v", gzipPath, err)
	}

	uncompressedStream, err := gzip.NewReader(bytes.NewBuffer(rawData))
	if err != nil {
		return fmt.Errorf("failed creating reader for gzip stream: %v", err)
	}
	tarReader := tar.NewReader(uncompressedStream)
	for {
		header, err := tarReader.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return fmt.Errorf("failed reading entry: %v", err)
		}

		rootHeaderPath := filepath.Join("/", header.Name)
		switch header.Typeflag {
		case tar.TypeDir:
			if err := os.Mkdir(rootHeaderPath, 0755); err != nil && !os.IsExist(err) {
				return fmt.Errorf("failed creating folder=%s, err=%v", rootHeaderPath, err)
			}
		case tar.TypeReg:
			outFile, err := os.Create(rootHeaderPath)
			if err != nil {
				return fmt.Errorf("failed creating file=%s, err=%v", rootHeaderPath, err)
			}
			defer outFile.Close()
			if _, err := io.Copy(outFile, tarReader); err != nil {
				return fmt.Errorf("failed copying file=%s, err=%v", rootHeaderPath, err)
			}
			if err := outFile.Chmod(0755); err != nil {
				return fmt.Errorf("failed changing perm from file=%s, err=%v", rootHeaderPath, err)
			}
		default:
			return fmt.Errorf("unknown type: %q in %s", header.Typeflag, header.Name)
		}
	}
	return nil
}

kubelet-unit.go

package nodeinit

import (
	"bytes"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"time"

	"github.com/coreos/go-systemd/dbus"
)

const (
	systemdpath          = "/etc/systemd/system"
	kubeletExtraArgsPath = "/etc/default/kubelet"
)

var KubeletUnitTemplate = []byte(
	`[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=http://kubernetes.io/docs/

[Service]
ExecStart=/opt/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target
`)

var KubeletDropInTemplate = []byte(
	`# Note: This dropin only works with kubeadm and kubelet v1.11+
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime,
# populating the KUBELET_KUBEADM_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort.
# Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead.
# KUBELET_EXTRA_ARGS should be sourced from this file.
EnvironmentFile=-/etc/default/kubelet
ExecStart=
ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
`)

var KubeletExtraArgsTemplate = []byte(`KUBELET_EXTRA_ARGS="{{ range .KubeletExtraArgs -}} {{ . }} {{- end }}"
`)

func StopKubelet() error {
	conn, err := dbus.New()
	if err != nil {
		return fmt.Errorf("failed getting dbus connection, err=%v", err)
	}
	defer conn.Close()
	response := make(chan string)
	_, err = conn.StopUnit("kubelet.service", "replace", response)
	if err != nil {
		return err
	}
	select {
	case status := <-response:
		return fmt.Errorf(status)
	case <-time.After(15 * time.Second):
		return fmt.Errorf("timeout(15s) on stopping coredns.service")
	}
}

// InstallKubelet copy the template files to systemd enable and start the service
func InstallKubelet(opts *CmdOptions) error {
	// Install Kubelet DropIn
	kubeletDropInPath := filepath.Join("/host", systemdpath, "kubelet.service.d")
	if err := os.MkdirAll(kubeletDropInPath, 0755); err != nil {
		return fmt.Errorf("failed creating dropin folder: %v", err)
	}
	dropInFilePath := filepath.Join(kubeletDropInPath, "10-kubeadm.conf")
	if err := ioutil.WriteFile(dropInFilePath, KubeletDropInTemplate, 0644); err != nil {
		return fmt.Errorf("failed writing dropin=%s, err=%v", dropInFilePath, err)
	}

	var kubeletExtraArgs []string
	for _, role := range opts.Roles() {
		kubeletExtraArgs = append(
			kubeletExtraArgs,
			fmt.Sprintf("--node-labels=node-role.kubernetes.io/%s ", role),
		)
	}
	kubeletExtraArgs = append(
		kubeletExtraArgs,
		fmt.Sprintf("--cloud-provider=%s ", opts.Kubelet.CloudProvider),
		fmt.Sprintf("--resolv-conf=%s", opts.Kubelet.ResolvConf),
	)
	if opts.Kubelet.ClusterDNS != "" {
		kubeletExtraArgs = append(
			kubeletExtraArgs,
			fmt.Sprintf(" --cluster-dns=%s", opts.Kubelet.ClusterDNS),
		)
	}
	if opts.NodeGroup != "" {
		kubeletExtraArgs = append(
			kubeletExtraArgs,
			fmt.Sprintf(" --node-labels=allspark.brainstemlabs.io/nodegroup=%s", opts.NodeGroup),
		)
	}
	tmpl, err := template.New("").Parse(string(KubeletExtraArgsTemplate))
	if err != nil {
		return fmt.Errorf("failed parsing kubelet extra args template, err=%v", err)
	}
	var buf bytes.Buffer
	if err := tmpl.Execute(&buf, map[string]interface{}{
		"KubeletExtraArgs": kubeletExtraArgs,
	}); err != nil {
		return fmt.Errorf("failed executing template, err=%v", err)
	}
	kubeletExtraArgsHostPath := filepath.Join("/host", kubeletExtraArgsPath)
	if err := ioutil.WriteFile(kubeletExtraArgsHostPath, buf.Bytes(), 0644); err != nil {
		return fmt.Errorf("falied writing kubelet extra args file, err=%v", err)
	}

	// Install Kubelet, Enable and Start
	kubeletHostSystemdPath := filepath.Join("/host", systemdpath, "kubelet.service")
	if err := ioutil.WriteFile(kubeletHostSystemdPath, KubeletUnitTemplate, 0644); err != nil {
		return fmt.Errorf("failed writing kubelet.service, err=%v", err)
	}
	conn, err := dbus.New()
	if err != nil {
		return fmt.Errorf("failed getting dbus connection, err=%v", err)
	}
	defer conn.Close()
	// if _, err := conn.ReloadOrRestartUnit("kubelet.service", "replace", nil); err != nil {
	// 	return fmt.Errorf("failed reloading systemd config, err=%v", err)
	// }
	kubeletSystemdPath := filepath.Join(systemdpath, "kubelet.service")
	_, changes, err := conn.EnableUnitFiles([]string{kubeletSystemdPath}, false, true)
	if err != nil {
		return fmt.Errorf("failed enabling unit, err=%v", err)
	}
	for _, chg := range changes {
		log.Printf("-> Created %s %s -> %s.", chg.Type, chg.Filename, chg.Destination)
	}
	if err := conn.Reload(); err != nil {
		log.Printf("failed reloading units, err=%v", err)
	}
	if _, err := conn.StartUnit("kubelet.service", "replace", nil); err != nil {
		return fmt.Errorf("failed starting unit, err=%v", err)
	}
	return nil
}

main.go

const CoreDNSLongText = `Add a systemd unit to run a CoreDNS for specific namespaces.
https://coredns.io/plugins/kubernetes/
`

const ToolsLongText = `Install CNI, crictl, kubeadm and kubelet to bootstrap a node with kubeadm
https://kubernetes.io/docs/setup/independent/install-kubeadm/
`

const nodeToolsPath = "/tmp/nodetools.tar.gz"

var o nodeinit.CmdOptions

func cmd() *cobra.Command {
	install := &cobra.Command{
		Use:   "install",
		Short: "Install essential tools for bootstraping a Kubernetes node",
	}
	coredns := &cobra.Command{
		Use:     "coredns",
		Short:   "Add a systemd unit to run a CoreDNS for specific namespaces",
		Long:    CoreDNSLongText,
		PostRun: cleanup,
		Run: func(cmd *cobra.Command, args []string) {
			log.Println("---> Stopping coredns.service ...")
			log.Printf("---> Status=%v", nodeinit.StopCoreDNS())
			log.Println("---> Installing CoreDNS ...")
			if err := nodeinit.InstallCoreDNS(&o); err != nil {
				log.Fatalf("failed installing coredns, err=%v", err)
			}
			log.Println("---> CoreDNS installed and configured with success!")
		},
	}
	coredns.Flags().StringVar(&o.CoreDNS.KubernetesEndpoint, "endpoint", "", "The URL for the remote k8s API endpoint")
	coredns.Flags().StringVar(&o.CoreDNS.Namespaces, "namespaces", "allspark", "Space separated list of namespace that will be exposed")
	coredns.MarkFlagRequired("endpoint")
	kubelet := &cobra.Command{
		Use:     "kubelet",
		Short:   "Install and configure Kubelet for master or node",
		PostRun: cleanup,
		Run: func(cmd *cobra.Command, args []string) {
			log.Println("---> Stopping kubelet.service ...")
			log.Printf("---> Status=%v", nodeinit.StopKubelet())
			log.Println("---> Installing Kubelet ...")
			if err := nodeinit.InstallKubelet(&o); err != nil {
				log.Fatalf("failed installing kubelet, err=%v", err)
			}
			log.Println("---> Kubelet installed and configured with success!")
		},
	}
	kubelet.Flags().StringVar(&o.Kubelet.CloudProvider, "cloud-provider", "", "The provider for cloud services.")
	kubelet.Flags().StringVar(&o.Kubelet.ResolvConf, "resolv-conf", "/etc/resolv.conf", "The provider for cloud services.")
	kubelet.Flags().StringVar(&o.Kubelet.ClusterDNS, "cluster-dns", "", "Comma-separated list of DNS server IP address.")
	kubelet.Flags().StringVar(&o.NodeRoles, "node-roles", "node", "A list of roles to add on the node - use comma to specify multiple roles.")
	kubelet.Flags().StringVar(&o.NodeGroup, "node-group", "", "Name of the node group.")

	tools := &cobra.Command{
		Use:     "tools",
		Long:    ToolsLongText,
		PostRun: cleanup,
		Run: func(cmd *cobra.Command, args []string) {
			if o.ShowVersionAndExit {
				version.PrintAndExit()
			}
			v := version.Get()
			log.Printf("Version=%s, Commit=%s, GoVersion=%s", v.Version, v.GitCommit, v.GoVersion)

			log.Println("---> Extracting node tools ...")
			// Install Node Tools
			// The tarball should be packed using relative paths, a / will be prefixed on extraction
			// Ex: $ tar -tvf <tarball>
			// -rwxr-xr-x  0 user group 57395147 Sep 17 18:29 host/opt/cni/bin/bridge
			// -rwxr-xr-x  0 user group 28466436 Jul 13 03:39 host/opt/bin/crictl
			// The example above will extract the files in /host/opt/cni/bin and /host/opt/bin/crictl
			if err := nodeinit.ExtractTarGz(nodeToolsPath); err != nil {
				log.Fatalf(err.Error())
			}
			log.Println("---> Node tools installed and configured with success!")
		},
	}

	root := cobra.Command{
		Use:     "nodeinit",
		Short:   "Bootstrap a node to join a Kubernetes cluster via kubeadm",
		PostRun: cleanup,
		Run: func(cmd *cobra.Command, args []string) {
			if o.ShowVersionAndExit {
				version.PrintAndExit()
			}
			cmd.Help()
		},
	}
	install.AddCommand(coredns, kubelet, tools)
	root.AddCommand(install)
	root.Flags().BoolVar(&o.ShowVersionAndExit, "version", false, "Print version and exit.")
	return &root
}

func cleanup(cmd *cobra.Command, args []string) { os.Remove(nodeToolsPath) }

func main() {
	cmd().Execute()
}