Vouch Authentication with Nginx and Caddy

__main__.py

from pathlib import Path
from base64 import b64encode
from secrets import token_urlsafe

from pulumi import Output, Config, ResourceOptions, export

from pulumi_azure_native import storage
from pulumi_azure_native.storage import StorageAccount, FileShare
from pulumi_azure_native.containerinstance import ContainerGroup, VolumeMountArgs, ContainerArgs
from pulumi_azure_native.resources import ResourceGroup

config = Config()
location = config.require("location")
env = config.require("env")
lowered_env = env.lower()
resource_group_config = config.require_object("resourceGroup")
vouch_config = config.require_object("vouch")

subdomain = f"app-{lowered_env}"
container_base = "centralus.azurecontainer.io"
domain = f"{subdomain}.{container_base}"

resource_group = ResourceGroup(
    f"resourceGroup{env}",
    location=location,
    resource_group_name=resource_group_config["name"],
    tags=resource_group_config["tags"],
)

storage_account = StorageAccount(
    f"pulumiStorage{env}",
    minimum_tls_version="TLS1_2",
    account_name=f"pulumistorageacct{lowered_env}",
    allow_blob_public_access=False,
    resource_group_name=resource_group.name,
    sku=storage.SkuArgs(name=storage.SkuName.STANDARD_LRS),
    kind=storage.Kind.STORAGE_V2,
)


def create_file_share(name, quota=1, protect=False):
    return FileShare(
        f"{name}-fileshare-{lowered_env}",
        opts=ResourceOptions(protect=protect),
        account_name=storage_account.name,
        resource_group_name=resource_group.name,
        share_quota=quota
    )

caddy_config_fileshare = create_file_share("caddy-config")
caddy_data_fileshare = create_file_share("caddy-data", protect=True)
vouch_secret_fileshare = create_file_share("vouch-secret")
prom_data_fileshare = create_file_share("prometheus-data", 5, protect=True)

primary_storage_account_key = Output.secret(
    Output.all(resource_group.name, storage_account.name).apply(
        lambda args: storage.list_storage_account_keys(
            resource_group_name=args[0], account_name=args[1]
        )
    ).apply(lambda keys: keys.keys[0].value)
)

def get_file_share_config(name, read_only=False):
    return {
        "share_name": name,
        "storage_account_name": storage_account.name,
        "read_only": read_only,
        "storage_account_key": primary_storage_account_key
    }

# https://hub.docker.com/_/caddy
# https://learn.microsoft.com/en-us/azure/container-instances/container-instances-container-group-automatic-ssl
caddy_container = ContainerArgs(
    name=f"caddy-{lowered_env}",
    image="caddy",
    resources={"requests": {"memory_in_gb": .5, "cpu": .5}},
    ports=[{"port": 80}, {"port": 443}],
    volume_mounts=[
        VolumeMountArgs(mount_path="/config", name="caddy-config", read_only=False),
        VolumeMountArgs(mount_path="/data", name="caddy-data", read_only=False),
        VolumeMountArgs(mount_path="/etc/caddy", name="caddyfile", read_only=False),
    ],
)

# https://hub.docker.com/_/nginx
nginx_container = ContainerArgs(
    name=f"nginx-{lowered_env}",
    image="nginx",
    resources={"requests": {"memory_in_gb": 1, "cpu": 1}},
    ports=[{"port": 8080}],
    volume_mounts=[
        VolumeMountArgs(mount_path="/etc/nginx/templates", name="nginx-templates", read_only=False),
    ],

    environment_variables=[
        {"name": "NGINX_HOST", "value": domain},
    ],
)

# https://github.com/vouch/vouch-proxy#running-from-docker
vouch_container = ContainerArgs(
    name=f"vouch-{lowered_env}",
    image="quay.io/vouch/vouch-proxy:latest",
    resources={"requests": {"memory_in_gb": 1, "cpu": 1}},
    ports=[{"port": 9091}],
    volume_mounts=[
        VolumeMountArgs(mount_path="/config/secret", name="vouch-secret", read_only=False),
        VolumeMountArgs(mount_path="/data", name="caddy-data", read_only=False),
    ],

    # https://github.com/vouch/vouch-proxy/blob/master/config/config.yml_example
    environment_variables=[
        {"name": "OAUTH_PROVIDER", "value": "azure"},
        {"name": "OAUTH_CLIENT_ID", "value": vouch_config["clientID"]},
        {"name": "OAUTH_CLIENT_SECRET", "value": vouch_config["clientSecret"]},
        {"name": "OAUTH_CALLBACK_URL", "value": f"https://{domain}/oauth2/auth"},
        {"name": "OAUTH_AUTH_URL", "value": "https://login.microsoftonline.com/{tenantID}/oauth2/v2.0/authorize".format(**vouch_config)},
        {"name": "OAUTH_TOKEN_URL", "value": "https://login.microsoftonline.com/{tenantID}/oauth2/v2.0/token".format(**vouch_config)},
        {"name": "OAUTH_USER_INFO_URL", "value": "https://graph.microsoft.com/oidc/userinfo"},
        {"name": "OAUTH_SCOPES", "value": "openid,profile,email"},
        {"name": "VOUCH_SESSION_KEY", "value": token_urlsafe(64)},
        {"name": "VOUCH_JWT_SECRET", "value": token_urlsafe(64)},
        {"name": "VOUCH_LOGLEVEL", "value": "debug"},
        {"name": "VOUCH_DOCUMENT_ROOT", "value": "/oauth2"},
        {"name": "VOUCH_TLS_PROFILE", "value": "intermediate"},
        {"name": "VOUCH_PORT", "value": "9091"},
        {"name": "VOUCH_TESTING", "value": False},
        {"name": "VOUCH_ALLOWALLUSERS", "value": True},
        {"name": "VOUCH_COOKIE_SECURE", "value": True},
        {"name": "VOUCH_COOKIE_SAMESITE", "value": "lax"},
        {"name": "VOUCH_COOKIE_DOMAIN", "value": domain},
        {"name": "VOUCH_TLS_CERT", "value": f"/data/caddy/certificates/acme-v02.api.letsencrypt.org-directory/{domain}/{domain}.crt"},
        {"name": "VOUCH_TLS_KEY", "value": f"/data/caddy/certificates/acme-v02.api.letsencrypt.org-directory/{domain}/{domain}.key"},
    ]
)

# https://hub.docker.com/r/bitnami/prometheus
prom_container = ContainerArgs(
    name=f"prometheus-{lowered_env}",
    image="docker.io/bitnami/prometheus:latest",
    resources={"requests": {"memory_in_gb": 1.5, "cpu": 1}},
    ports=[{"port": 9090}],
    volume_mounts=[
        VolumeMountArgs(
            mount_path="/opt/bitnami/prometheus/data",
            name="prometheus-data",
            read_only=False,
        ),
        VolumeMountArgs(
            mount_path="/opt/bitnami/prometheus/conf",
            name="prometheus-config",
            read_only=False,
        ),
    ]
)

def get_container_volume(name, fileshare=None, secret=None, secret_text=None, file_name=None):
    volume = {"name": name}

    if secret or secret_text:
        volume["secret"] = {}

        if secret:
            path = Path(secret)

            with path.open("rb") as f:
                file_name = path.name
                file_text = f.read()
        elif file_name:
            file_text = secret_text.encode()
        else:
            raise ValueError("Must provide either secret or secret_text and file_name")

        volume["secret"][file_name] = b64encode(file_text).decode("ascii")
    elif fileshare:
        volume["azure_file"] = get_file_share_config(fileshare.name)
    else:
        raise ValueError("Must provide either fileshare, secret, or secret_text and file_name")

    return volume

caddyfile_text = f"""
{{
    email {config.require("email")}
    acme_ca https://acme-v02.api.letsencrypt.org/directory
}}
{domain} {{
    reverse_proxy :8080 {{
        header_up X-Forwarded-Port 443
    }}
    header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
}}"""

container_volumes = [
    get_container_volume("caddy-config", caddy_config_fileshare),
    get_container_volume("caddy-data", caddy_data_fileshare),
    get_container_volume("caddyfile", secret_text=caddyfile_text, file_name="Caddyfile"),
    get_container_volume("nginx-templates", secret="default.conf.template"),
    get_container_volume("vouch-secret", vouch_secret_fileshare),
    get_container_volume("prometheus-data", prom_data_fileshare),
    get_container_volume("prometheus-config", secret="prometheus.yml"),
]

container_group = ContainerGroup(
    f"prometheusContainerGroup{env}",
    containers=[
        caddy_container,
        nginx_container,
        vouch_container,
        prom_container,
    ],
    ip_address={
        "ports": [{"port": 80}, {"port": 443}],
        "type": "Public",
        "dns_name_label": subdomain,
        "auto_generated_domain_name_label_scope": "TenantReuse"
    },
    os_type="Linux",
    resource_group_name=resource_group.name,
    container_group_name=f"prometheus-container-group-{lowered_env}",
    location=resource_group.location,
    restart_policy="OnFailure",
    volumes=container_volumes,
)

export("fqdn", container_group.ip_address.fqdn)
export("publicIP", container_group.ip_address.ip)

default.conf.template

upstream prometheus {
  server 127.0.0.1:9090;
  keepalive 20;
}

upstream vouch {
  server 127.0.0.1:9091;
  keepalive 20;
}

server {
  listen 8080 default_server;
  listen [::]:8080 default_server ipv6only=on;
  http2 on;
  server_name ${NGINX_HOST};

  # get client ip addresses
  # https://serverfault.com/a/414166
  set_real_ip_from 127.0.0.1;
  set_real_ip_from 192.168.2.1;
  real_ip_header X-Forwarded-For;
  real_ip_recursive on;
  add_header "X-Forwarded-For" "$http_x_forwarded_for, $realip_remote_addr";
  add_header "X-Real-IP" $remote_addr;

  # This location serves vouch/validate
  location = /oauth2/validate {
    # CORS preflight requests dont contain a cookie
    # https://stackoverflow.com/questions/41760128/cookies-not-sent-on-options-requests
    if ($request_method = 'OPTIONS') {
      return 204 no-content;
    }

    # forward the /validate request to Vouch Proxy
    proxy_pass https://vouch/oauth2/validate;

    # be sure to pass the original host header
    proxy_set_header Host $http_host;

    # Vouch Proxy only acts on the request headers
    proxy_pass_request_body off;
    proxy_set_header Content-Length "";

    # pass IP headers
    proxy_set_header X-Forwarded-For "$http_x_forwarded_for, $realip_remote_addr";
    proxy_set_header X-Real-IP $remote_addr;
    proxy_pass_request_headers on;

    # optionally add X-Vouch-User as returned by Vouch Proxy along with the request
    auth_request_set $auth_resp_x_vouch_user $upstream_http_x_vouch_user;

    # these return values are used by the @error401 call
    auth_request_set $auth_resp_jwt $upstream_http_x_vouch_jwt;
    auth_request_set $auth_resp_err $upstream_http_x_vouch_err;
    auth_request_set $auth_resp_failcount $upstream_http_x_vouch_failcount;
  }

  # This location serves all of the paths vouch uses except validate
  # /oauth2/login, /oauth2/logout, /oauth2/auth, /oauth2/auth/$STATE, etc
  location /oauth2 {
    proxy_pass https://vouch;
    proxy_set_header Host $http_host;

    # pass IP headers
    proxy_set_header X-Forwarded-For "$http_x_forwarded_for, $realip_remote_addr";
    proxy_set_header X-Real-IP $remote_addr;
    proxy_pass_request_headers on;
  }

  # if validate returns `401 not authorized` then forward the request to the error401block
  error_page 401 = @error401;

  location @error401 {
    # redirect to Vouch Proxy for login
    return 302 $scheme://${NGINX_HOST}/oauth2/login?url=https://$http_host$request_uri&vouch-failcount=$auth_resp_failcount&X-Vouch-Token=$auth_resp_jwt&error=$auth_resp_err;
    # you usually *want* to redirect to Vouch running behind the same Nginx config proteced by https
    # but to get started you can just forward the end user to the port that vouch is running on
  }

  # proxy pass authorized requests to your service
  location / {
    # send all requests to the `/oauth2/validate` endpoint for authorization
    auth_request /oauth2/validate;

    # forward authorized requests to prometheus
    proxy_pass http://prometheus;

    auth_request_set $auth_resp_x_vouch_user $upstream_http_x_vouch_user;
    auth_request_set $auth_resp_x_vouch_idp_accesstoken $upstream_http_x_vouch_idp_accesstoken;
    auth_request_set $auth_resp_x_vouch_idp_idtoken $upstream_http_x_vouch_idp_idtoken;
    auth_request_set $auth_resp_x_vouch_idp_claims_groups $upstream_http_x_vouch_idp_claims_groups;
    auth_request_set $auth_resp_x_vouch_idp_claims_given_name $upstream_http_x_vouch_idp_claims_given_name;

    # be sure to pass the original host header
    proxy_set_header Host $http_host;
    proxy_set_header X-Vouch-User $auth_resp_x_vouch_user;
    proxy_set_header X-Vouch-IdP-AccessToken $auth_resp_x_vouch_idp_accesstoken;
    proxy_set_header X-Vouch-IdP-IdToken $auth_resp_x_vouch_idp_idtoken;
    proxy_set_header X-Vouch-IdP-Claims-Groups $auth_resp_x_vouch_idp_claims_groups;
    proxy_set_header X-Vouch-IdP-Claims-Given_Name $auth_resp_x_vouch_idp_claims_given_name;

    # pass IP headers
    proxy_set_header X-Forwarded-For "$http_x_forwarded_for, $realip_remote_addr";
    proxy_set_header X-Real-IP $remote_addr;
    proxy_pass_request_headers on;
  }
}

prometheus.yml

global:
  scrape_interval: 60s
  evaluation_interval: 60s

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

Pulumi.dev.yaml

config:
  env: Dev
  resourceGroup:
    name: <name>
    tags:
      key: value
  vouch:
    clientID: <ID>
    clientSecret:
      secure: <secure>
    tenantID: <ID>

Pulumi.yaml

name: <name>
runtime:
  name: python
  options:
    virtualenv: venv
description: <description>
config:
  location: <location>
  email: <email>