Enable ECR (AWS) registries for Spinnaker with Kubernetes provider

config.yml

# A part of the Halyard config file declaring the ECR registries.
# There can be multiple registries, each in different AWS account.
# In this example there are 3 "stages" accounts - dev, stage & live.
# NOTE: The declared password files must exist and provide valid base64 encoded values,
#       otherwise Halayrd will endup with an exception during deployment.
#       The values can be fake, they will be updated later by the Kubernetes Job (see 2-nd attached file).
# NOTE: replace ${YOUR_DEV_AWS_ACCOUNT_ID} ${YOUR_DEV_AWS_REGION}
#       with appropriate values (same for STAGE & LIVE).

dockerRegistry:
  enabled: true
  accounts:
  - name: ecr-dev
    address: https://${YOUR_DEV_AWS_ACCOUNT_ID}.dkr.ecr.${YOUR_DEV_AWS_REGION}.amazonaws.com
    username: AWS
    passwordFile: /opt/passwords/ecr-registry-shared.pass
  - name: ecr-stage
    address: https://${YOUR_STAGE_AWS_ACCOUNT_ID}.dkr.ecr.${YOUR_STAGE_AWS_REGION}.amazonaws.com
    username: AWS
    passwordFile: /opt/passwords/ecr-registry-stage.pass
  - name: ecr-live
    address: https://${YOUR_LIVE_AWS_ACCOUNT_ID}.dkr.ecr.${YOUR_LIVE_AWS_REGION}.amazonaws.com
    username: AWS
    passwordFile: /opt/passwords/ecr-registry-live.pass
   

k8s-cron-job.yml

# The K8s CronJob which task is to refresh the AWS ECR tokens used by Spinnaker to access registries
# and store them in K8s Secrets created by Halyard deployment.
# Note that the K8s secrets names are changing for each Halyard deployment.
# The shell script for the K8s CronJob gets the related K8s Secrets names from the `spin-clouddriver`
# and `spin-clouddriver-bootstrap` ReplicaSets related Pods.
# The ReplicaSets and Pods names are resolved in runtime, so the solution will work independently
# of subsequent Halyard deployments (chanding their names with a particular version sufix).
# Additionally a ServiceAccount with a proper RBAC is created, which allows for the listing ReplicaSets,
# Pods an ConfigMaps, executing commands in Pods (due to access the Spinnaker configuration)
# and listing & patching Secrets.

# NOTE: replace ${YOUR_DEV_AWS_ACCOUNT_ID} ${YOUR_DEV_AWS_REGION} and ${YOUR_DEV_AWS_CROSS_ACCOUNT_ECR_ACCESS_ROLE_ARN}
#       with appropriate values (same for STAGE & LIVE).

apiVersion: v1
kind: ServiceAccount
metadata:
  name: ecr-token-updater-serviceaccount
  namespace: spinnaker
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: ecr-token-updater-clusterrole
rules:
  - apiGroups:
      - ""
      - "extensions"
    resources:
      - configmaps
      - replicasets
      - pods
    verbs:
      - get
      - list
  - apiGroups:
      - ""
    resources:
      - pods/exec
    verbs:
      - create
  - apiGroups:
      - ""
    resources:
      - secrets
    verbs:
      - list
      - get
      - patch
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: ecr-token-updater-clusterrole-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: ecr-token-updater-clusterrole
subjects:
  - kind: ServiceAccount
    name: ecr-token-updater-serviceaccount
    namespace: spinnaker
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: ecr-token-updater-script
  namespace: spinnaker
data:
  update-ecr-tokens.sh: |-
    #!/bin/sh

    set -e

    POD_NAME=$(kubectl get pods -n spinnaker --selector=cluster=spin-clouddriver -o json  | jq -r '.items[] | [.metadata.name] | sort[]' | tail -1)

    function setPassword {
      SECRET_FIELD_NAME=$(kubectl exec $POD_NAME -c spin-clouddriver -n spinnaker -- cat /opt/spinnaker/config/clouddriver.yml | grep -e "passwordFile.*${1}.*" -o | awk -F "/" '{print $NF}')
      if [ -z ${5+x} ]; then
        ECR_PASSWORD=$(aws ecr get-authorization-token --registry-ids=${2} --region ${4} --output text --query 'authorizationData[].authorizationToken' | base64 -d | cut -d: -f2 | base64 | tr -d ' \t\n\r\f');
      else
        SESSION=$(aws sts assume-role --role-arn ${5} --role-session-name ecr-shared-access --query 'Credentials');
        SECRET_ACCESS_KEY=$(echo $SESSION | jq -r '.SecretAccessKey');
        ACCESS_KEY_ID=$(echo $SESSION | jq -r '.AccessKeyId');
        SESSION_TOKEN=$(echo $SESSION | jq -r '.SessionToken');
        ECR_PASSWORD=$(AWS_SESSION_TOKEN=$SESSION_TOKEN AWS_ACCESS_KEY_ID=$ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$SECRET_ACCESS_KEY aws ecr get-authorization-token --registry-ids=${2} --region ${4} --output text --query 'authorizationData[].authorizationToken' | base64 -d | cut -d: -f2 | base64 | tr -d ' \t\n\r\f');
      fi
      kubectl patch secret $3 -n spinnaker -p "{\"data\": {\"${SECRET_FIELD_NAME}\": \"${ECR_PASSWORD}\"}}"
    }

    function updateSecretForReplicaSet {
      REPLICA_SET_NAME=$(kubectl get replicasets -n spinnaker --selector=${1} -o json  | jq -r '.items[] | [.metadata.name] | sort[]' | tail -1)
      PASSWORD_VOLUME_NAME=$(kubectl get replicaset $REPLICA_SET_NAME -n spinnaker -o json | jq -r \
        ".spec.template.spec.containers[] | select(.name==\"${2}\") | .volumeMounts[] | select(.mountPath==\"/root/.hal/default/staging/dependencies\") | .name")

      setPassword dev ${YOUR_DEV_AWS_ACCOUNT_ID} $PASSWORD_VOLUME_NAME ${YOUR_DEV_AWS_REGION} ${YOUR_DEV_AWS_CROSS_ACCOUNT_ECR_ACCESS_ROLE_ARN}
      setPassword stage ${YOUR_STAGE_AWS_ACCOUNT_ID} $PASSWORD_VOLUME_NAME ${YOUR_STAGE_AWS_REGION} ${YOUR_STAGE_AWS_CROSS_ACCOUNT_ECR_ACCESS_ROLE_ARN}
      setPassword live ${YOUR_LIVE_AWS_ACCOUNT_ID} $PASSWORD_VOLUME_NAME ${YOUR_LIVE_AWS_REGION}
    }

    updateSecretForReplicaSet cluster=spin-clouddriver spin-clouddriver
    updateSecretForReplicaSet load-balancer-spin-clouddriver-bootstrap=true spin-clouddriver-bootstrap
---
apiVersion: batch/v1beta1
kind: CronJob
metadata:
  name: ecr-token-updater
  namespace: spinnaker
spec:
  schedule: "* */6 * * *"
  successfulJobsHistoryLimit: 1
  failedJobsHistoryLimit: 1
  jobTemplate:
    spec:
      template:
        spec:
          serviceAccountName: ecr-token-updater-serviceaccount
          volumes:
          - name: ecr-token-updater-script
            configMap:
              name: ecr-token-updater-script
              defaultMode: 0755
          containers:
          - name: ecr-token-updater
            image: myobplatform/aws-jq-kubectl:0.0.1
            command:
            - "/ecr/update-ecr-tokens.sh"
            volumeMounts:
            - name: ecr-token-updater-script
              mountPath: "/ecr"
              readOnly: true
          restartPolicy: Never