env.go

package vkubelet

import (
	"context"
	"fmt"
	"sort"
	"strings"

	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	apivalidation "k8s.io/apimachinery/pkg/util/validation"
	"k8s.io/client-go/tools/record"

	"github.com/virtual-kubelet/virtual-kubelet/log"
	"github.com/virtual-kubelet/virtual-kubelet/manager"
)

const (
	// ReasonOptionalConfigMapNotFound is the reason used in events emitted when an optional configmap is not found.
	ReasonOptionalConfigMapNotFound = "OptionalConfigMapNotFound"
	// ReasonOptionalConfigMapKeyNotFound is the reason used in events emitted when an optional configmap key is not found.
	ReasonOptionalConfigMapKeyNotFound = "OptionalConfigMapKeyNotFound"
	// ReasonFailedToReadOptionalConfigMap is the reason used in events emitted when an optional configmap could not be read.
	ReasonFailedToReadOptionalConfigMap = "FailedToReadOptionalConfigMap"

	// ReasonOptionalSecretNotFound is the reason used in events emitted when an optional secret is not found.
	ReasonOptionalSecretNotFound = "OptionalSecretNotFound"
	// ReasonOptionalSecretKeyNotFound is the reason used in events emitted when an optional secret key is not found.
	ReasonOptionalSecretKeyNotFound = "OptionalSecretKeyNotFound"
	// ReasonFailedToReadOptionalSecret is the reason used in events emitted when an optional secret could not be read.
	ReasonFailedToReadOptionalSecret = "FailedToReadOptionalSecret"

	// ReasonMandatoryConfigMapNotFound is the reason used in events emitted when an mandatory configmap is not found.
	ReasonMandatoryConfigMapNotFound = "MandatoryConfigMapNotFound"
	// ReasonMandatoryConfigMapKeyNotFound is the reason used in events emitted when an mandatory configmap key is not found.
	ReasonMandatoryConfigMapKeyNotFound = "MandatoryConfigMapKeyNotFound"
	// ReasonFailedToReadMandatoryConfigMap is the reason used in events emitted when an mandatory configmap could not be read.
	ReasonFailedToReadMandatoryConfigMap = "FailedToReadMandatoryConfigMap"

	// ReasonMandatorySecretNotFound is the reason used in events emitted when an mandatory secret is not found.
	ReasonMandatorySecretNotFound = "MandatorySecretNotFound"
	// ReasonMandatorySecretKeyNotFound is the reason used in events emitted when an mandatory secret key is not found.
	ReasonMandatorySecretKeyNotFound = "MandatorySecretKeyNotFound"
	// ReasonFailedToReadMandatorySecret is the reason used in events emitted when an mandatory secret could not be read.
	ReasonFailedToReadMandatorySecret = "FailedToReadMandatorySecret"

	// ReasonInvalidEnvironmentVariableNames is the reason used in events emitted when a configmap/secret referenced in a ".spec.containers[*].envFrom" field contains invalid environment variable names.
	ReasonInvalidEnvironmentVariableNames = "InvalidEnvironmentVariableNames"
)

// populateEnvironmentVariables populates the environment of each container (and init container) in the specified pod.
// TODO Make this the single exported function of a "pkg/environment" package in the future.
func populateEnvironmentVariables(ctx context.Context, pod *corev1.Pod, rm *manager.ResourceManager, recorder record.EventRecorder) error {
	// Populate each init container's environment.
	for idx := range pod.Spec.InitContainers {
		if err := populateContainerEnvironment(ctx, pod, &pod.Spec.InitContainers[idx], rm, recorder); err != nil {
			return err
		}
	}
	// Populate each container's environment.
	for idx := range pod.Spec.Containers {
		if err := populateContainerEnvironment(ctx, pod, &pod.Spec.Containers[idx], rm, recorder); err != nil {
			return err
		}
	}
	return nil
}

// populateContainerEnvironment populates the environment of a single container in the specified pod.
func populateContainerEnvironment(ctx context.Context, pod *corev1.Pod, container *corev1.Container, rm *manager.ResourceManager, recorder record.EventRecorder) error {
	// Create an "environment map" based on the value of the specified container's ".envFrom" field.
	envFrom, err := makeEnvironmentMapBasedOnEnvFrom(ctx, pod, container, rm, recorder)
	if err != nil {
		return err
	}
	// Create an "environment map" based on the value of the specified container's ".env" field.
	env, err := makeEnvironmentMapBasedOnEnv(ctx, pod, container, rm, recorder)
	if err != nil {
		return err
	}
	// Empty the container's ".envFrom" field and replace its ".env" field with the final, merged environment.
	// Values in "env" (sourced from ".env") will override any values with the same key defined in "envFrom" (sourced from ".envFrom").
	// This is in accordance with what the Kubelet itself does.
	// https://github.com/kubernetes/kubernetes/blob/v1.13.1/pkg/kubelet/kubelet_pods.go#L557-L558
	container.EnvFrom = []corev1.EnvFromSource{}
	container.Env = mergeEnvironments(envFrom, env)
	return nil
}

// makeEnvironmentMapBasedOnEnvFrom returns a map representing the resolved environment of the specified container after being populated from the entries in the ".envFrom" field.
func makeEnvironmentMapBasedOnEnvFrom(ctx context.Context, pod *corev1.Pod, container *corev1.Container, rm *manager.ResourceManager, recorder record.EventRecorder) (map[string]string, error) {
	// Create a map to hold the resulting environment.
	res := make(map[string]string, 0)
	// Iterate over "envFrom" references in order to populate the environment.
loop:
	for _, envFrom := range container.EnvFrom {
		switch {
		// Handle population from a configmap.
		case envFrom.ConfigMapRef != nil:
			ef := envFrom.ConfigMapRef
			// Check whether the configmap reference is optional.
			// This will control whether we fail when unable to read the configmap.
			optional := ef.Optional != nil && *ef.Optional
			// Try to grab the referenced configmap.
			m, err := rm.GetConfigMap(ef.Name, pod.Namespace)
			if err != nil {
				// We couldn't fetch the configmap.
				// However, if the configmap reference is optional we should not fail.
				if optional {
					if errors.IsNotFound(err) {
						recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalConfigMapNotFound, "configmap %q not found", ef.Name)
					} else {
						log.G(ctx).Warnf("failed to read configmap %q: %v", ef.Name, err)
						recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadOptionalConfigMap, "failed to read configmap %q", ef.Name)
					}
					// Continue on to the next reference.
					continue loop
				}
				// At this point we know the configmap reference is mandatory.
				// Hence, we should return a meaningful error.
				if errors.IsNotFound(err) {
					recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatoryConfigMapNotFound, "configmap %q not found", ef.Name)
					return nil, fmt.Errorf("configmap %q not found", ef.Name)
				}
				recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadMandatoryConfigMap, "failed to read configmap %q", ef.Name)
				return nil, fmt.Errorf("failed to fetch configmap %q: %v", ef.Name, err)
			}
			// At this point we have successfully fetched the target configmap.
			// Iterate over the keys defined in the configmap and populate the environment accordingly.
			// https://github.com/kubernetes/kubernetes/blob/v1.13.1/pkg/kubelet/kubelet_pods.go#L581-L595
			invalidKeys := make([]string, 0)
		mKeys:
			for key, val := range m.Data {
				// If a prefix has been defined, prepend it to the environment variable's name.
				if len(envFrom.Prefix) > 0 {
					key = envFrom.Prefix + key
				}
				// Make sure that the resulting key is a valid environment variable name.
				// If it isn't, it should be appended to the list of invalid keys and skipped.
				if errMsgs := apivalidation.IsEnvVarName(key); len(errMsgs) != 0 {
					invalidKeys = append(invalidKeys, key)
					continue mKeys
				}
				// Add the key and its value to the environment.
				res[key] = val
			}
			// Report any invalid keys.
			if len(invalidKeys) > 0 {
				sort.Strings(invalidKeys)
				recorder.Eventf(pod, corev1.EventTypeWarning, ReasonInvalidEnvironmentVariableNames, "keys [%s] from configmap %s/%s were skipped since they are invalid as environment variable names", strings.Join(invalidKeys, ", "), m.Namespace, m.Name)
			}
		// Handle population from a secret.
		case envFrom.SecretRef != nil:
			ef := envFrom.SecretRef
			// Check whether the secret reference is optional.
			// This will control whether we fail when unable to read the secret.
			optional := ef.Optional != nil && *ef.Optional
			// Try to grab the referenced secret.
			s, err := rm.GetSecret(ef.Name, pod.Namespace)
			if err != nil {
				// We couldn't fetch the secret.
				// However, if the secret reference is optional we should not fail.
				if optional {
					if errors.IsNotFound(err) {
						recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalSecretNotFound, "secret %q not found", ef.Name)
					} else {
						log.G(ctx).Warnf("failed to read secret %q: %v", ef.Name, err)
						recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadOptionalSecret, "failed to read secret %q", ef.Name)
					}
					// Continue on to the next reference.
					continue loop
				}
				// At this point we know the secret reference is mandatory.
				// Hence, we should return a meaningful error.
				if errors.IsNotFound(err) {
					recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatorySecretNotFound, "secret %q not found", ef.Name)
					return nil, fmt.Errorf("secret %q not found", ef.Name)
				}
				recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadMandatorySecret, "failed to read secret %q", ef.Name)
				return nil, fmt.Errorf("failed to fetch secret %q: %v", ef.Name, err)
			}
			// At this point we have successfully fetched the target secret.
			// Iterate over the keys defined in the secret and populate the environment accordingly.
			// https://github.com/kubernetes/kubernetes/blob/v1.13.1/pkg/kubelet/kubelet_pods.go#L581-L595
			invalidKeys := make([]string, 0)
		sKeys:
			for key, val := range s.Data {
				// If a prefix has been defined, prepend it to the environment variable's name.
				if len(envFrom.Prefix) > 0 {
					key = envFrom.Prefix + key
				}
				// Make sure that the resulting key is a valid environment variable name.
				// If it isn't, it should be appended to the list of invalid keys and skipped.
				if errMsgs := apivalidation.IsEnvVarName(key); len(errMsgs) != 0 {
					invalidKeys = append(invalidKeys, key)
					continue sKeys
				}
				// Add the key and its value to the environment.
				res[key] = string(val)
			}
			// Report any invalid keys.
			if len(invalidKeys) > 0 {
				sort.Strings(invalidKeys)
				recorder.Eventf(pod, corev1.EventTypeWarning, ReasonInvalidEnvironmentVariableNames, "keys [%s] from secret %s/%s were skipped since they are invalid as environment variable names", strings.Join(invalidKeys, ", "), s.Namespace, s.Name)
			}
		}
	}
	// Return the populated environment.
	return res, nil
}

// makeEnvironmentMapBasedOnEnv returns a map representing the resolved environment of the specified container after being populated from the entries in the ".env" field.
func makeEnvironmentMapBasedOnEnv(ctx context.Context, pod *corev1.Pod, container *corev1.Container, rm *manager.ResourceManager, recorder record.EventRecorder) (map[string]string, error) {
	// Create a map to hold the resolved environment variables.
	res := make(map[string]string, len(container.Env))
	// Iterate over environment variables in order to populate the map.
loop:
	for _, env := range container.Env {
		switch {
		// Handle values that have been directly provided.
		case env.Value != "":
			res[env.Name] = env.Value
			continue loop
		// Handle population from a configmap key.
		case env.ValueFrom != nil && env.ValueFrom.ConfigMapKeyRef != nil:
			// The environment variable must be set from a configmap.
			vf := env.ValueFrom.ConfigMapKeyRef
			// Check whether the key reference is optional.
			// This will control whether we fail when unable to read the requested key.
			optional := vf != nil && *vf.Optional
			// Try to grab the referenced configmap.
			m, err := rm.GetConfigMap(vf.Name, pod.Namespace)
			if err != nil {
				// We couldn't fetch the configmap.
				// However, if the key reference is optional we should not fail.
				if optional {
					if errors.IsNotFound(err) {
						recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalConfigMapNotFound, "skipping optional envvar %q: configmap %q not found", env.Name, vf.Name)
					} else {
						log.G(ctx).Warnf("failed to read configmap %q: %v", vf.Name, err)
						recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadOptionalConfigMap, "skipping optional envvar %q: failed to read configmap %q", env.Name, vf.Name)
					}
					// Continue on to the next reference.
					continue loop
				}
				// At this point we know the key reference is mandatory.
				// Hence, we should return a meaningful error.
				if errors.IsNotFound(err) {
					recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatoryConfigMapNotFound, "configmap %q not found", vf.Name)
					return nil, fmt.Errorf("configmap %q not found", vf.Name)
				}
				recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadMandatoryConfigMap, "failed to read configmap %q", vf.Name)
				return nil, fmt.Errorf("failed to read configmap %q: %v", vf.Name, err)
			}
			// At this point we have successfully fetched the target configmap.
			// We must now try to grab the requested key.
			var (
				keyExists bool
				keyValue  string
			)
			if keyValue, keyExists = m.Data[vf.Key]; !keyExists {
				// The requested key does not exist.
				// However, we should not fail if the key reference is optional.
				if optional {
					// Continue on to the next reference.
					recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalConfigMapKeyNotFound, "skipping optional envvar %q: key %q does not exist in configmap %q", env.Name, vf.Key, vf.Name)
					continue loop
				}
				// At this point we know the key reference is mandatory.
				// Hence, we should fail.
				recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatoryConfigMapKeyNotFound, "key %q does not exist in configmap %q", vf.Key, vf.Name)
				return nil, fmt.Errorf("configmap %q doesn't contain the %q key required by pod %s", vf.Name, vf.Key, pod.Name)
			}
			// Populate the environment variable and continue on to the next reference.
			res[env.Name] = keyValue
			continue loop
		// Handle population from a secret key.
		case env.ValueFrom != nil && env.ValueFrom.SecretKeyRef != nil:
			vf := env.ValueFrom.SecretKeyRef
			// Check whether the key reference is optional.
			// This will control whether we fail when unable to read the requested key.
			optional := vf != nil && *vf.Optional
			// Try to grab the referenced secret.
			s, err := rm.GetSecret(vf.Name, pod.Namespace)
			if err != nil {
				// We couldn't fetch the secret.
				// However, if the key reference is optional we should not fail.
				if optional {
					if errors.IsNotFound(err) {
						recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalSecretNotFound, "skipping optional envvar %q: secret %q not found", env.Name, vf.Name)
					} else {
						log.G(ctx).Warnf("failed to read secret %q: %v", vf.Name, err)
						recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadOptionalSecret, "skipping optional envvar %q: failed to read secret %q", env.Name, vf.Name)
					}
					// Continue on to the next reference.
					continue loop
				}
				// At this point we know the key reference is mandatory.
				// Hence, we should return a meaningful error.
				if errors.IsNotFound(err) {
					recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatorySecretNotFound, "secret %q not found", vf.Name)
					return nil, fmt.Errorf("secret %q not found", vf.Name)
				}
				recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadMandatorySecret, "failed to read secret %q", vf.Name)
				return nil, fmt.Errorf("failed to read secret %q: %v", vf.Name, err)
			}
			// At this point we have successfully fetched the target secret.
			// We must now try to grab the requested key.
			var (
				keyExists bool
				keyValue  []byte
			)
			if keyValue, keyExists = s.Data[vf.Key]; !keyExists {
				// The requested key does not exist.
				// However, we should not fail if the key reference is optional.
				if optional {
					// Continue on to the next reference.
					recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalSecretKeyNotFound, "skipping optional envvar %q: key %q does not exist in secret %q", env.Name, vf.Key, vf.Name)
					continue loop
				}
				// At this point we know the key reference is mandatory.
				// Hence, we should fail.
				recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatorySecretKeyNotFound, "key %q does not exist in secret %q", vf.Key, vf.Name)
				return nil, fmt.Errorf("secret %q doesn't contain the %q key required by pod %s", vf.Name, vf.Key, pod.Name)
			}
			// Populate the environment variable and continue on to the next reference.
			res[env.Name] = string(keyValue)
			continue loop
		// Handle population from a field (downward API).
		case env.ValueFrom != nil && env.ValueFrom.FieldRef != nil:
			//Get Something
			vf := env.ValueFrom.FieldRef
			switch vf.FieldPath {
			case "metadata.name":
				res[env.Name] = pod.ObjectMeta.Name
			case "status.PodIP":
				res[env.Name] = pod.Status.PodIP
			case "metadata.namespace":
				res[env.Name] = pod.ObjectMeta.Namespace
			case "spec.serviceAccountName":
				res[env.Name] = pod.Spec.ServiceAccountName
			case "spec.nodeName":
				res[env.Name] = pod.Spec.NodeName
			}
			continue loop
		// Handle population from a resource request/limit.
		case env.ValueFrom != nil && env.ValueFrom.ResourceFieldRef != nil:
			// TODO Implement populating resource requests.
			continue loop
		}
	}
	// Return the populated environment.
	return res, nil
}

// mergeEnvironments creates the final environment for a container by merging "envFrom" and "env".
// Values in "env" override any values with the same key defined in "envFrom".
// This is in accordance with what the Kubelet itself does.
// https://github.com/kubernetes/kubernetes/blob/v1.13.1/pkg/kubelet/kubelet_pods.go#L557-L558
func mergeEnvironments(envFrom map[string]string, env map[string]string) []corev1.EnvVar {
	tmp := make(map[string]string, 0)
	res := make([]corev1.EnvVar, 0)
	for key, val := range envFrom {
		tmp[key] = val
	}
	for key, val := range env {
		tmp[key] = val
	}
	for key, val := range tmp {
		res = append(res, corev1.EnvVar{
			Name:  key,
			Value: val,
		})
	}
	return res
}