ArgoCD configuration

argocd-rbac.jsonnet

local registry = import 'registry/index.jsonnet';

// Compute the rbac rules that should be configured in ArgoCD, using the service
// registry to determine who should be given what permissions.
//
// This module can be evaluated concretely, which may help to debug the rbac
// list for ArgoCD.
{
  _config:: {
    // Operator grants- anyone with admin or operator should be permitted to
    // perform these actions.
    operator_grants: [
      'sync',
      'action/apps/Deployment/restart',
      'action/batch/Job/delete',
      'action/core/Pod/delete',
    ],

    // Viewer is very limited in ArgoCD, but we provide structure that can
    // easily expand if we need.
    viewer_grants: ['get'],
  },

  // Admin powers are defined per-cluster, as we consider each cluster a
  // security boundary. If you assume that granting admin into Kubernetes
  // increases the exposure to security threats, which seems sensible, then we
  // want to ensure that the type of dangerous powers we provide to admins needs
  // to be consistent within the same cluster.
  //
  // This field pre-computes the grants that admins should be provided for each
  // cluster, so we can reference them when building the ArgoCD RBAC rules.
  cluster_admin_grants: {
    [cluster.spec.name]: std.prune(std.flattenArrays([
      (
        if std.member(cluster.spec.permissions, 'superpowers')
        then ['update']
        else []
      ),
    ]))
    for cluster in registry.clusters
  },

  // List of rules for all ArgoCD services, to be templated into the
  // argocd-rbac-cm configmap.
  rules:
    local build(target) =
      local build(emails, grants) = [
        {
          email: email,
          grant: grant,
          application: std.strReplace(target.spec.id, '/', '-'),
          project: '%s-%s' % [target.spec.context, target.spec.namespace],
        }
        for email in emails
        for grant in grants
      ];

      local emails = target.environment.spec.rbac.transitive.emails(registry);
      std.flattenArrays([
        // Viewer and operator grants are unchanged between clusters.
        build(emails.viewer, $._config.viewer_grants),
        build(emails.operator, $._config.operator_grants),
        // Admin grants are conditional on the cluster we exist within, which is
        // why we have to compute them from the cluster_admin_grants map.
        build(emails.admin, $.cluster_admin_grants[target.spec.context]),
      ]);

    std.flattenArrays([
      build(target)
      for target in registry.registry.enumerate.targets
      if target.spec.strategy == 'Argo'
    ]),
}

argocd.libsonnet

local crds = import './crds.json';
local argoproj = import 'crds/k8s.libsonnet';
local k = import 'gcausal.libsonnet';

// Computed rbac rules for templating into the argocd-rbac-cm.
local rbac = import './argocd-rbac.jsonnet';

// Use the registry to generate Application manifests for all argo deployment
// targets
local registry = import 'registry/index.jsonnet';

// Extract targets from the registry, adding a target.service and
// target.environment reference
local services = registry.registry.enumerate.services;
local environments = registry.registry.enumerate.environments;
local targets = registry.registry.enumerate.targets;

local containsString(string, substring) =
  std.length(std.findSubstr(substring, string)) > 0;

/*
ArgoCD provides authentication using a dex server, configured with a Google
connector. Dex makes additional requests to GSuite to resolve group membership
for each user that logs in.

[email protected]

The above service account is used to provide dex with the ability to query the
GSuite directory API. We don't have a good story around injecting a service
account key for this account, which is why it's done by hand. If we ever
reconfigure argo, we'll need to ensure the key from terraform is manually
re-added as under the `credentials.json` key of the
argocd-dex-credentials secret.
*/
k + argoproj + {
  local cfg = self._config,
  _config:: {
    namespace: 'argocd',
    hostname: 'argocd.gocardless.io',

    // This should be set to the official anu image unless testing. Use the
    // alpine image as the scratch doesn't have cp.
    utopia_image: 'eu.gcr.io/gc-containers/gocardless/anu:%s' % [
      'v14.0.0-alpine',
    ],

    // Look in the Kubernetes secret called argocd-secret for the key
    // dex.google.clientSecret. This is provisioned by hand, but you
    // can find this value at:
    //
    // https://console.cloud.google.com/apis/credentials/oauthclient/961152851217-k3tsak9dduga63tf5j31b2dvg0dem7l3.apps.googleusercontent.com?project=gc-prd-effc
    clientID: '961152851217-k3tsak9dduga63tf5j31b2dvg0dem7l3.apps.googleusercontent.com',
    clientSecret: '$dex.google.clientSecret',

    autosync_snitch_id: error '_config.autosync_snitch_id required',
  },

  local namespace = $.core.v1.namespace,
  local application = $.argoproj.v1alpha1.application,
  local configMap = $.core.v1.configMap,
  local deployment = $.apps.v1.deployment,
  local container = deployment.mixin.spec.template.spec.containersType,
  local initContainer = deployment.mixin.spec.template.spec.initContainersType,
  local volumeMount = container.volumeMountsType,
  local service = $.core.v1.service,
  local ingress = $.networking.v1beta1.ingress,
  local ingressRule = ingress.mixin.spec.rulesType,
  local ingressTls = ingress.mixin.spec.tlsType,
  local httpIngressPath = ingressRule.mixin.http.pathsType,

  namespace:
    namespace.new() +
    namespace.mixin.metadata.withName(cfg.namespace),

  // We should use argo to manage argo application config, instead of relying on
  // people manually creating these in the UI. The application configuration
  // lives inside of utopia, and this just bootstraps that management process.
  registry:
    application.new() +
    application.mixin.metadata.withName('registry') +
    application.mixin.metadata.withNamespace(cfg.namespace) +
    application.mixin.spec.destination.withNamespace(cfg.namespace) +
    application.mixin.spec.destination.withServer('https://kubernetes.default.svc') +
    application.mixin.spec.withProject('default') +
    application.mixin.spec.source.withPath('utopia') +
    application.mixin.spec.source.withRepoUrl('[email protected]:gocardless/anu') +
    application.mixin.spec.source.plugin.withName('utopia') +
    application.mixin.spec.source.plugin.withEnv([
      { name: 'FILE', value: 'applications.jsonnet' },
    ]) +
    // We adjust the environment of each application via our deployment process,
    // so we don't want Argo to try managing it.
    application.mixin.spec.withIgnoreDifferences([{
      group: 'argoproj.io',
      kind: 'Application',
      jsonPointers: [
        '/spec/source/plugin/env',
      ],
    }]),

  resources: [
    resource {
      metadata+: {
        namespace: cfg.namespace,
      },
    } +
    // This configmap contains generic argocd configuration
    (if resource.kind == 'ConfigMap' && resource.metadata.name == 'argocd-cm' then
       configMap.withData({
         // Enable use of status badges so we can show sync status on the README
         // of each application.
         'statusbadge.enabled': 'true',
         // Provide a user account for the dispatcher service, allowing the
         // dispatcher to authenticate via an auth token
         'accounts.dispatcher': 'apiKey',
         // Provide a user account for the script that automatically performs the initial
         // sync of applications into non-privileged clusters.
         'accounts.appsyncer': 'apiKey',
         // Configure dex to perform an augmented Google OAuth flow, one that
         // performs look-ups against GSuite to add group claims.
         'dex.config': std.manifestYamlDoc({
           connectors: [{
             type: 'google',
             id: 'google',
             name: 'Google',
             config: {
               clientID: cfg.clientID,
               clientSecret: cfg.clientSecret,
               redirectURI: 'https://%s/api/dex/callback' % cfg.hostname,
               hostedDomains: ['gocardless.com'],
               serviceAccountFilePath: '/var/run/secrets/google/credentials.json',
               // Impersonate this user, it just has to be a GSuite admin
               adminEmail: '[email protected]',
             },
           }],
         }),
         // Add utopia as a plugin in Argo. This makes the utopia show command a
         // first-class concept in Argo-land. FILE is often left unset, as we
         // can detect the correct file using ARGOCD_APP_NAME.
         configManagementPlugins: std.manifestYamlDoc([{
           name: 'utopia',
           generate: {
             command: ['bash', '-c'],
             args: ['utopia show --revision "${REVISION}" --file "${FILE}"'],
           },
         }]),
       })
     else {}) +
    // Configure policies and rbac controls here:
    // https://argoproj.github.io/argo-cd/operator-manual/rbac/
    (if resource.kind == 'ConfigMap' && resource.metadata.name == 'argocd-rbac-cm' then
       configMap.withData({
         // Anyone in the core-infrastructure-team Google group should be granted
         // admin, while anyone in engineering should be able to read things.
         // Anyone else should be denied access entirely.
         //
         // On-callers outside of Core Infra should also have admin access, so
         // the infrastructure-on-call group is also granted that role.
         //
         // The dispatcher user is our CD pipeline, and should be able to do
         // most things on applications (other than delete and create them).
         'policy.csv': std.join('\n', std.flattenArrays([
           [
             'g, [email protected], role:admin',
             'g, [email protected], role:admin',
             'g, [email protected], role:readonly',
             'p, dispatcher, projects, get, *, allow',
             'p, dispatcher, applications, *, */*, allow',
             'p, appsyncer, projects, get, *, allow',
             'p, appsyncer, applications, get, */*, allow',
             'p, appsyncer, applications, sync, */*, allow',
             'p, appsyncer, applications, update, */*, allow',
           ],
           // See argocd-rbac.jsonnet for more detail on how we calculate the
           // rbac permissions per application. This code is just for templating
           // them.
           std.map(
             function(rule)
               'p, %(email)s, applications, %(grant)s, %(project)s/%(application)s, allow' % rule,
             rbac.rules
           ),
         ])),
       })
     else {}) +
    // The controller outputs kube-state-metric-like Prometheus metrics about
    // the state of each application, which will help us alert on things.
    (if resource.kind == 'Deployment' && resource.metadata.name == 'argocd-application-controller' then
       deployment.mapContainersWithName(
         'argocd-application-controller', function(c)
           c + container.withPorts([{
             containerPort: 8082,
             name: 'metrics',
           }])
       )
     else {}) +
    // Patch the dex server with our Google credentials (see top-level comment
    // for auth explanation)
    (if resource.kind == 'Deployment' && resource.metadata.name == 'argocd-dex-server' then
       deployment.mixin.spec.template.spec.withVolumesMixin({
         name: 'credentials',
         secret: {
           secretName: 'argocd-dex-credentials',
         },
       }) +
       deployment.mapContainersWithName(
         'dex', function(c)
           c + container.withVolumeMountsMixin(
             volumeMount.new() +
             volumeMount.withName('credentials') +
             volumeMount.withMountPath('/var/run/secrets/google/credentials.json') +
             volumeMount.withSubPath('credentials.json'),
           ),
       )
     else {}) +
    // Patch the repo server deployment to provide access to the utopia binary,
    // which we'll use as an argo plugin
    (if resource.kind == 'Deployment' && resource.metadata.name == 'argocd-repo-server' then
       deployment.mixin.spec.template.spec.withVolumesMixin({ name: 'tools', emptyDir: {} }) +
       deployment.mixin.spec.template.spec.withInitContainersMixin(
         initContainer.new() +
         initContainer.withName('install-utopia') +
         initContainer.withImage(cfg.utopia_image) +
         initContainer.withCommand([
           'cp',
           '-v',
           '/bin/utopia',
           '/tools/utopia',
         ]) +
         initContainer.withVolumeMounts(
           volumeMount.new() +
           volumeMount.withName('tools') +
           volumeMount.withMountPath('/tools'),
         ),
       ) +
       deployment.mapContainersWithName(
         'argocd-repo-server', function(c)
           c + container.withVolumeMountsMixin(
             volumeMount.new() +
             volumeMount.withName('tools') +
             volumeMount.withMountPath('/usr/local/bin/utopia') +
             volumeMount.withSubPath('utopia'),
           ),
       )
     else {}) +
    // Patch the service to add the app-protocols annotation, otherwise GLB will
    // health check in HTTP and be turned away.
    (if resource.kind == 'Service' && resource.metadata.name == 'argocd-server' then
       service.mixin.metadata.withAnnotationsMixin({
         'cloud.google.com/app-protocols': '{"https":"HTTPS"}',
       }) +
       // Exposed via ingress, must be NodePort
       service.mixin.spec.withType('NodePort')
     else {})
    for resource in crds
  ],

  // Argo creates its own certificates on boot, and I couldn't find an easy way
  // to convince it to use a different key pair.
  //
  // It also hosts several protocols all on the same port: HTTPS for the web UI,
  // and grpc via HTTP2 for the API.
  //
  // This setup tries to keep the end-to-end TLS (with the self-serve
  // certificates) while presenting a valid certificate for the external URL to
  // our users.
  //
  // That means we:
  //  - Terminate TLS at the GLB ingress, but register the backend as a HTTPS
  //    protocol
  //  - Expose the grpc protocol via a load balancer service
  ingress:
    ingress.new() +
    ingress.mixin.metadata.withName('argocd-server') +
    ingress.mixin.metadata.withNamespace(cfg.namespace) +
    ingress.mixin.metadata.withAnnotations({
      'kubernetes.io/ingress.allow-http': 'false',
      'cert-manager.io/cluster-issuer': 'letsencrypt',
      'external-dns.alpha.kubernetes.io/hostname': cfg.hostname,
    }) +
    ingress.mixin.spec.withTls(
      ingressTls.new() +
      ingressTls.withHosts(cfg.hostname) +
      ingressTls.withSecretName('argocd-server-tls'),
    ) +
    ingress.mixin.spec.withRules(
      ingressRule.new() +
      ingressRule.withHost(cfg.hostname) +
      ingressRule.mixin.http.withPaths(
        httpIngressPath.new() +
        httpIngressPath.mixin.backend.withServicePort('https') +
        httpIngressPath.mixin.backend.withServiceName('argocd-server'),
      ),
    ),

  // This provides access to the grpc API with certificates that won't have this
  // hostname. People using the argocd CLI will just have to deal with this, for
  // now.
  service_grpc:
    service.new() +
    service.mixin.metadata.withName('argocd-server-grpc') +
    service.mixin.metadata.withNamespace(cfg.namespace) +
    service.mixin.metadata.withAnnotations({
      'external-dns.alpha.kubernetes.io/hostname': 'grpc.%s' % $._config.hostname,
    }) +
    service.mixin.spec.withType('LoadBalancer') +
    service.mixin.spec.withSelector({
      'app.kubernetes.io/name': 'argocd-server',
    }) +
    service.mixin.spec.withPorts({
      name: 'https',
      port: 443,
      targetPort: 8080,
    }),

  syncer_config_map:
    local cm = $.core.v1.configMap;

    cm.new('application-syncer-script') +
    cm.mixin.metadata.withNamespace($._config.namespace) +
    cm.mixin.metadata.withLabels({ app: 'argocd-application-syncer' }) +
    $.core.v1.configMap.withData({
      'syncer.rb': (importstr 'sync_apps_on_argocd.rb'),
    }),

  syncer_deployment:
    local args = {
      clusters:
        std.set(std.map(
          function(t) t.spec.context,
          std.filter(
            // We want to allow all lab clusters to auto-sync in argo,
            // this will filter out the specified clusters by mathcing
            // for the existence of 'lab' in the cluster context name.
            function(target) containsString(target.spec.context, 'lab'),
            targets,
          ),
        )),
    };

    local app =
      container.new('app', 'ruby:2.5-alpine') +
      container.withArgsMixin(
        [
          'ruby',
          '/app/syncer.rb',
          '--clusters',
          std.join(',', args.clusters),
          '--prd',
          'true',
          '--snitch_id',
          cfg.autosync_snitch_id,
        ]
      ) +
      container.withEnvMixin(
        {
          name: 'ARGOCD_SERVER',
          value: 'argocd-server.%s.svc.cluster.local' % cfg.namespace,
        },
      ) +
      container.withEnvMixin(
        {
          name: 'ARGOCD_OPTS',
          // We use a self-signed certificate to serve the ArgoCD API, so for
          // the moment must disable TLS verification.
          value: '--insecure',
        },
      ) +
      container.withEnvMixin(
        $.gc.container.env.withValueFrom.new() +
        $.gc.container.env.withValueFrom.withName('ARGOCD_AUTH_TOKEN') +
        // This secret has been manually created with:
        // `argocd account generate-token --account appsyncer`
        $.gc.container.env.withValueFrom.mixin.secret.withName('argocd-application-syncer-credentials') +
        $.gc.container.env.withValueFrom.mixin.secret.withKey('token'),
      ) +
      $.util.resourcesRequests('100m', '256Mi') +
      $.util.resourcesLimits('200m', '256Mi');

    deployment.new('argocd-application-syncer', containers=[app]) +
    deployment.mixin.metadata.withName('application-syncer') +
    $.gc.deployer.mixin.withPodLabelsAndSelector({ app: 'argocd-application-syncer', role: 'worker' }) +
    $.util.configMapVolumeMount($.syncer_config_map, '/app', { readOnly: true }) +
    deployment.mixin.metadata.withNamespace(cfg.namespace),
}