argo

README.md

platform-argo

infrastructure component for managing argo and related resources

Getting Started

0. prereqs

• jsonnet
• kubecfg
• terraform

1. terraform

# render terraform variables
$ jsonnet -m . teams.jsonnet

# initialize backend
$ terraform init

# apply
$ terraform apply --var-file terraform.auto.tfvars.json

2. kubecfg

# apply k8s manifests
$ kubecfg update --gc-tag argo manifest.jsonnet

argo-events.yml

---
# All argo-events services are bound to the "argo-events" service account. 
# In RBAC enabled setups, this SA is bound to specific roles.
apiVersion: v1
kind: ServiceAccount
metadata:
  name: argo-events-sa
  namespace: argo-events
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: argo-events-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: argo-events-role
subjects:
  - kind: ServiceAccount
    name: argo-events-sa
    namespace: argo-events
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: argo-events-role
rules:
  - apiGroups:
    - apiextensions.k8s.io
    - apiextensions.k8s.io/v1beta1
    verbs:
      - create
      - delete
      - deletecollection
      - get
      - list
      - patch
      - update
      - watch
    resources:
      - customresourcedefinitions
  - apiGroups:
    - argoproj.io
    verbs:
      - create
      - delete
      - deletecollection
      - get
      - list
      - patch
      - update
      - watch
    resources:
      - workflows
      - workflows/finalizers
      - gateways
      - gateways/finalizers
      - sensors
      - sensors/finalizers
  - apiGroups:
    - ""
    resources:
      - pods
      - pods/exec
      - configmaps
      - secrets
      - services
      - events
      - persistentvolumeclaims
    verbs:
      - create
      - get
      - list
      - watch
      - update
      - patch
      - delete
  - apiGroups:
    - "batch"
    resources:
      - jobs
    verbs:
      - create
      - get
      - list
      - watch
      - update
      - patch
      - delete
  - apiGroups:
    - "apps/v1"
    - "apps/v1beta2"
    - "apps/v1beta1"
    resources:
      - deployments
    verbs:
      - create
      - get
      - list
      - watch
      - update
      - patch
      - delete
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
  rbac.authorization.k8s.io/aggregate-to-admin: "true"
  name: argo-events-aggregate-to-admin
rules:
  - apiGroups:
    - argoproj.io
    resources:
      - gateways
      - gateways/finalizers
      - sensors
      - sensors/finalizers
    verbs:
      - create
      - delete
      - deletecollection
      - get
      - list
      - patch
      - update
      - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
  rbac.authorization.k8s.io/aggregate-to-edit: "true"
  name: argo-events-aggregate-to-edit
rules:
  - apiGroups:
    - argoproj.io
    resources:
      - gateways
      - gateways/finalizers
      - sensors
      - sensors/finalizers
    verbs:
      - create
      - delete
      - deletecollection
      - get
      - list
      - patch
      - update
      - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
  rbac.authorization.k8s.io/aggregate-to-view: "true"
  name: argo-events-aggregate-to-view
rules:
  - apiGroups:
    - argoproj.io
    resources:
      - gateways
      - gateways/finalizers
      - sensors
      - sensors/finalizers
    verbs:
      - get
      - list
      - watch
---
# Define a "sensor" custom resource definition
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: sensors.argoproj.io
spec:
  group: argoproj.io
  names:
  kind: Sensor
  listKind: SensorList
  plural: sensors
  singular: sensor
  scope: Namespaced
  version: v1alpha1
---
# Define a "gateway" custom resource definition
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: gateways.argoproj.io
spec:
  group: argoproj.io
  names:
  kind: Gateway
  listKind: GatewayList
  plural: gateways
  singular: gateway
  scope: Namespaced
  version: v1alpha1
---
# The sensor-controller configmap includes configuration information for the sensor-controller
# To watch sensors created in different namespace than the controller is deployed in, remove the namespace: argo-events.
# Similarly to watch sensors created in specific namespace, change to namespace: <your_namespace>
apiVersion: v1
kind: ConfigMap
metadata:
  name: sensor-controller-configmap
data:
  config: |
    instanceID: argo-events
    namespace: argo-events
---
# The gateway-controller configmap includes configuration information for the gateway-controller
# To watch gateways created in different namespace than the controller is deployed in, remove the namespace: argo-events.
# Similarly to watch gateways created in specific namespace, change to namespace: <your_namespace>
apiVersion: v1
kind: ConfigMap
metadata:
  name: gateway-controller-configmap
data:
  config: |
    instanceID: argo-events
    namespace: argo-events
---
# The sensor-controller listens for changes on the sensor CRD and creates sensor executor jobs
apiVersion: apps/v1
kind: Deployment
metadata:
  name: sensor-controller
spec:
  replicas: 1
  selector:
  matchLabels:
    app: sensor-controller
  template:
  metadata:
    labels:
    app: sensor-controller
  spec:
    serviceAccountName: argo-events-sa
    containers:
    - name: sensor-controller
    image: argoproj/sensor-controller
    imagePullPolicy: Always
    env:
      - name: SENSOR_NAMESPACE
        valueFrom:
          fieldRef:
          fieldPath: metadata.namespace
      - name: SENSOR_CONFIG_MAP
        value: sensor-controller-configmap
---
# The gateway-controller listens for changes on the gateway CRD and creates gateway
apiVersion: apps/v1
kind: Deployment
metadata:
  name: gateway-controller
spec:
  replicas: 1
  selector:
  matchLabels:
    app: gateway-controller
  template:
  metadata:
    labels:
    app: gateway-controller
  spec:
    serviceAccountName: argo-events-sa
    containers:
    - name: gateway-controller
    image: argoproj/gateway-controller
    imagePullPolicy: Always
    env:
      - name: GATEWAY_NAMESPACE
        valueFrom:
          fieldRef:
          fieldPath: metadata.namespace
      - name: GATEWAY_CONTROLLER_CONFIG_MAP
        value: gateway-controller-configmap 

argo.yml

---
# This is an auto-generated file. DO NOT EDIT
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: workflows.argoproj.io
spec:
  group: argoproj.io
  names:
    kind: Workflow
    plural: workflows
    shortNames:
    - wf
  scope: Namespaced
  version: v1alpha1
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: argo
  namespace: argo
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: argo-ui
  namespace: argo
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    rbac.authorization.k8s.io/aggregate-to-admin: "true"
  name: argo-aggregate-to-admin
rules:
- apiGroups:
  - argoproj.io
  resources:
  - workflows
  - workflows/finalizers
  verbs:
  - create
  - delete
  - deletecollection
  - get
  - list
  - patch
  - update
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    rbac.authorization.k8s.io/aggregate-to-edit: "true"
  name: argo-aggregate-to-edit
rules:
- apiGroups:
  - argoproj.io
  resources:
  - workflows
  - workflows/finalizers
  verbs:
  - create
  - delete
  - deletecollection
  - get
  - list
  - patch
  - update
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    rbac.authorization.k8s.io/aggregate-to-view: "true"
  name: argo-aggregate-to-view
rules:
- apiGroups:
  - argoproj.io
  resources:
  - workflows
  - workflows/finalizers
  verbs:
  - get
  - list
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: argo-cluster-role
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - pods/exec
  verbs:
  - create
  - get
  - list
  - watch
  - update
  - patch
  - delete
- apiGroups:
  - ""
  resources:
  - configmaps
  verbs:
  - get
  - watch
  - list
- apiGroups:
  - ""
  resources:
  - persistentvolumeclaims
  verbs:
  - create
  - delete
- apiGroups:
  - argoproj.io
  resources:
  - workflows
  - workflows/finalizers
  verbs:
  - get
  - list
  - watch
  - update
  - patch
  - delete
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: argo-ui-cluster-role
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - pods/exec
  - pods/log
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - secrets
  verbs:
  - get
- apiGroups:
  - argoproj.io
  resources:
  - workflows
  verbs:
  - get
  - list
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: argo-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: argo-cluster-role
subjects:
- kind: ServiceAccount
  name: argo
  namespace: argo
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: argo-ui-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: argo-ui-cluster-role
subjects:
- kind: ServiceAccount
  name: argo-ui
  namespace: argo
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: workflow-controller-configmap
  namespace: argo
---
apiVersion: v1
kind: Service
metadata:
  name: argo-ui
  namespace: argo
spec:
  ports:
  - port: 80
    targetPort: 8001
  selector:
    app: argo-ui
---
apiVersion: apps/v1beta2
kind: Deployment
metadata:
  name: argo-ui
  namespace: argo
spec:
  selector:
    matchLabels:
      app: argo-ui
  template:
    metadata:
      labels:
        app: argo-ui
    spec:
      containers:
      - env:
        - name: ARGO_NAMESPACE
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
        - name: IN_CLUSTER
          value: "true"
        - name: ENABLE_WEB_CONSOLE
          value: "false"
        - name: BASE_HREF
          value: /
        image: argoproj/argoui:v2.4.0
        name: argo-ui
      serviceAccountName: argo-ui
---
apiVersion: apps/v1beta2
kind: Deployment
metadata:
  name: workflow-controller
  namespace: argo
spec:
  selector:
    matchLabels:
      app: workflow-controller
  template:
    metadata:
      labels:
        app: workflow-controller
    spec:
      containers:
      - args:
        - --configmap
        - workflow-controller-configmap
        - --executor-image
        - argoproj/argoexec:v2.2.1
        command:
        - workflow-controller
        image: argoproj/workflow-controller:v2.2.1
        name: workflow-controller
      serviceAccountName: argo

main.tf

################################################################################
## Terraform
################################################################################

terraform {
  required_version = "~> 0.12.6"
}

################################################################################
## Variables
################################################################################

variable "kubernetes_worker_role" {
  description = "kubernetes worker role"
  type        = string
}

variable "region" {
  description = "aws region"
  type        = string
}

variable "teams" {
  description = "map of teams by name"
  type = map(object({
    name = string,
    iam_policies = object({
      managed = map(string),
      custom = map(list(object({
        actions   = list(string),
        effect    = string,
        resources = list(string),
        conditions = list(object({
          test     = string,
          variable = string,
          value    = list(any),
        })),
      }))),
    }),
  }))
}

################################################################################
## Constants
################################################################################

locals {
  policies_custom = flatten([
    for name, team in var.teams : [
      for alias, statements in team.iam_policies.custom : {
        id         = "${name}-${alias}"
        team       = name,
        alias      = alias,
        statements = statements
      }
    ]
  ])

  policies_managed = flatten([
    for name, team in var.teams : [
      for alias, arn in team.iam_policies.managed : {
        id    = "${name}-${alias}"
        team  = name,
        alias = alias,
        arn   = arn
      }
    ]
  ])

  tags = {
    BillingGroup = "v7"
    component    = "platform-argo"
    terraform    = "true"
    workspace    = terraform.workspace
  }
}

################################################################################
## Providers
################################################################################

provider "aws" {
  region = var.region
}

################################################################################
## Resources
################################################################################

# define trust policy
data "aws_iam_policy_document" "argo_trust_policy" {
  statement {
    actions = ["sts:AssumeRole"]
    effect  = "Allow"
    principals {
      type        = "AWS"
      identifiers = [var.kubernetes_worker_role]
    }
    principals {
      type        = "Service"
      identifiers = ["ec2.amazonaws.com"]
    }
  }
}

# provision argo role for each team
resource "aws_iam_role" "argo" {
  for_each           = var.teams
  name               = "argo-${each.value.name}"
  description        = "argo workflow role for ${each.value.name} team"
  assume_role_policy = data.aws_iam_policy_document.argo_trust_policy.json
  tags               = local.tags
}

# provision custom argo policies
resource "aws_iam_policy" "argo_custom" {
  for_each = {
    for policy in local.policies_custom : policy.id => policy
  }

  name        = "argo-${each.value.id}"
  description = "argo role policy for ${each.value.team} team"
  policy      = data.aws_iam_policy_document.argo_custom[each.value.id].json
}

# define custom argo policies
data "aws_iam_policy_document" "argo_custom" {
  for_each = {
    for policy in local.policies_custom : policy.id => policy
  }

  dynamic "statement" {
    for_each = each.value.statements

    content {
      actions   = statement.value.actions
      effect    = statement.value.effect
      resources = statement.value.resources

      dynamic "condition" {
        for_each = statement.value.conditions

        content {
          test     = condition.value.test
          variable = condition.value.variable
          values   = condition.value.values
        }
      }
    }
  }
}

# attach custom argo policies to argo role
resource "aws_iam_role_policy_attachment" "argo_custom" {
  for_each = {
    for policy in local.policies_custom : policy.id => policy
  }

  role       = aws_iam_role.argo[each.value.team].id
  policy_arn = aws_iam_policy.argo_custom[each.value.id].arn
}

# attach managed argo policies to argo role
resource "aws_iam_role_policy_attachment" "argo_managed" {
  for_each = {
    for policy in local.policies_managed : policy.id => policy
  }

  role       = aws_iam_role.argo[each.value.team].id
  policy_arn = each.value.arn
}


################################################################################
## Outputs
################################################################################

output "roles" {
  value = {
    for team in var.teams : team.name => aws_iam_role.argo[team.name]
  }
}

manifest.jsonnet

local kube = import 'https://raw.githubusercontent.com/bitnami-labs/kube-libsonnet/master/kube.libsonnet';
local kubecfg = import 'kubecfg.libsonnet';
local teams = import 'teams.jsonnet';

function(labels={})
  kube.List() {
    items: std.flattenArrays([
      // import argo manifest
      std.filterMap($.filterObjects, $.setNamespace('argo'), kubecfg.parseYaml(importstr 'argo.yml')),

      // import argo-events manifest
      std.filterMap($.filterObjects, $.setNamespace('argo-events'), kubecfg.parseYaml(importstr 'argo-events.yml')),
    ]) + std.flattenArrays([
      // provision team-scoped resources
      $.teamObjects(teams.data[teamName] { name: teamName })
      for teamName in std.objectFields(teams.data)
    ]),

    // valid k8s filter function
    filterObjects(obj)::
      obj != null && std.objectHas(obj, 'apiVersion') && std.objectHas(obj, 'kind'),

    // factory function for namespace mapper fn
    setNamespace(ns)::
      function(obj)
        if std.length(std.find(obj.kind, ['ClusterRole', 'ClusterRoleBinding', 'CustomResourceDefinition', 'Namespace'])) > 0 then
          obj
        else
          obj {
            metadata+: {
              namespace: ns,
            },
          },

    // generate team-scoped kubernetes objects
    teamObjects(team)::
      // create team-scoped argo namespace
      local ns = kube.Namespace('argo-%s' % team.name);

      // create workflow service account
      local sa = kube.ServiceAccount('workflow') {
        metadata+: {
          namespace: ns.metadata.name,
        },
      };

      // grant workflow service account rbac
      local rb = kube.RoleBinding('workflow') {
        metadata+: {
          namespace: ns.metadata.name,
        },
        subjects_: [sa],
        roleRef_: {
          kind: 'ClusterRole',
          metadata: {
            name: 'edit',
          },
        },
      };

      [
        ns,
        sa,
        rb,
      ],
  }

teams.jsonnet

{
  data:: {
    'data-services': {
      iam_policies: {
        custom: {
          test: [
            {
              actions: ['s3:ListBuckets'],
              effect: 'Allow',
              resources: ['*'],
              conditions: [],
            },
          ],
        },
        managed: {
          AmazonAPIGatewayAdministrator: 'arn:aws:iam::aws:policy/AmazonAPIGatewayAdministrator',
        },
      },
    },
    sre: {
      iam_policies: {
        managed: {
          AdministratorAccess: 'arn:aws:iam::aws:policy/AdministratorAccess',
        },
      },
    },
  },

  'terraform.auto.tfvars.json': {
    kubernetes_worker_role: 'test',
    region: 'us-east-1',
    teams: std.mapWithKey(function(key, value) value {
      name: key,
      iam_policies+: {
        custom+: {},
        managed+: {},
      },
    }, $.data),
  },
}

terraform.auto.tfvars.json

{
   "kubernetes_worker_role": "arn:aws:iam::301764520140:role/data-services-terraform-root",
   "region": "us-east-1",
   "teams": {
      "data-services": {
         "iam_policies": {
            "custom": {
               "test": [
                  {
                     "actions": [
                        "s3:ListBuckets"
                     ],
                     "conditions": [ ],
                     "effect": "Allow",
                     "resources": [
                        "*"
                     ]
                  }
               ]
            },
            "managed": {
               "AmazonAPIGatewayAdministrator": "arn:aws:iam::aws:policy/AmazonAPIGatewayAdministrator"
            }
         },
         "name": "data-services"
      },
      "sre": {
         "iam_policies": {
            "custom": { },
            "managed": {
               "AdministratorAccess": "arn:aws:iam::aws:policy/AdministratorAccess"
            }
         },
         "name": "sre"
      }
   }
}