PgDog RSS-retention repro stand (jemalloc dirty pages / background_thread) — see pgdogdev/pgdog issue

pgdog-rss-repro-stand.yaml

# PgDog RSS-retention reproduction stand (Kubernetes).
# See pgdogdev/pgdog issue: RSS not returned to OS after bursts of large messages.
#
#   kubectl apply -f pgdog-rss-repro-stand.yaml
#   # generate a ~3MB client-message load file inside the loadgen pod:
#   POD=$(kubectl -n pgdog-lt get pod -l app=loadgen -o name)
#   kubectl -n pgdog-lt exec $POD -- sh -c \
#     'printf "SELECT octet_length('\''%s'\'');\n" "$(head -c 3000000 /dev/zero | tr "\0" a)" > /tmp/bigq.sql'
#   # drive 500 clients for 80s against the DEFAULT pgdog (balloons and sticks):
#   kubectl -n pgdog-lt exec $POD -- env PGPASSWORD=ltpass pgbench -n -c 500 -j 16 -T 80 \
#     -f /tmp/bigq.sql "host=pgdog-lt-default port=6432 user=lt dbname=lt sslmode=disable"
#   # repeat against pgdog-lt-bg (background_thread:true) -> RSS returns to baseline.
#   # sample RSS:
#   kubectl -n pgdog-lt exec deploy/pgdog-lt-default -- awk '/VmRSS/{print int($2/1024)" MB"}' /proc/1/status
apiVersion: v1
kind: Namespace
metadata:
  name: pgdog-lt
---
# throwaway postgres backend
apiVersion: apps/v1
kind: Deployment
metadata:
  name: pg-lt
  namespace: pgdog-lt
spec:
  replicas: 1
  selector: { matchLabels: { app: pg-lt } }
  template:
    metadata: { labels: { app: pg-lt } }
    spec:
      containers:
        - name: postgres
          image: postgres:18
          args: ["-c","max_connections=700","-c","shared_buffers=512MB"]
          env:
            - { name: POSTGRES_USER, value: "lt" }
            - { name: POSTGRES_PASSWORD, value: "ltpass" }
            - { name: POSTGRES_DB, value: "lt" }
            - { name: PGDATA, value: "/var/lib/postgresql/data/pgdata" }
          ports: [ { containerPort: 5432 } ]
          resources:
            requests: { cpu: "500m", memory: "512Mi" }
            limits: { cpu: "2", memory: "2Gi" }
          volumeMounts: [ { name: data, mountPath: /var/lib/postgresql/data } ]
          readinessProbe:
            exec: { command: ["pg_isready","-U","lt","-d","lt"] }
            initialDelaySeconds: 5
            periodSeconds: 5
      volumes: [ { name: data, emptyDir: {} } ]
---
apiVersion: v1
kind: Service
metadata: { name: pg-lt, namespace: pgdog-lt }
spec:
  selector: { app: pg-lt }
  ports: [ { port: 5432, targetPort: 5432 } ]
---
# pgdog config (shared by both variants)
apiVersion: v1
kind: ConfigMap
metadata: { name: pgdog-lt-config, namespace: pgdog-lt }
data:
  pgdog.toml: |
    [general]
    host = "0.0.0.0"
    port = 6432
    default_pool_size = 10
    min_pool_size = 1
    query_parser = "off"
    passthrough_auth = "disabled"
    healthcheck_port = 8080
    openmetrics_port = 9090
    openmetrics_namespace = "pgdog_"
    connect_attempts = 10

    [[databases]]
    name = "lt"
    host = "pg-lt.pgdog-lt.svc.cluster.local"
    port = 5432
    database_name = "lt"
---
apiVersion: v1
kind: Secret
metadata: { name: pgdog-lt-users, namespace: pgdog-lt }
stringData:
  users.toml: |
    [[users]]
    name = "lt"
    database = "lt"
    password = "ltpass"
---
# pgdog DEFAULT (no MALLOC_CONF) — RSS balloons and stays stuck
apiVersion: apps/v1
kind: Deployment
metadata: { name: pgdog-lt-default, namespace: pgdog-lt }
spec:
  replicas: 1
  selector: { matchLabels: { app: pgdog-lt-default } }
  template:
    metadata: { labels: { app: pgdog-lt-default } }
    spec:
      containers:
        - name: pgdog
          image: ghcr.io/pgdogdev/pgdog:0.1.48
          command: ["/usr/local/bin/pgdog"]
          args: ["--config","/etc/pgdog/pgdog.toml","--users","/etc/secrets/pgdog/users.toml"]
          ports: [ { containerPort: 6432 }, { containerPort: 9090 } ]
          resources:
            requests: { cpu: "500m", memory: "200Mi" }
            limits: { cpu: "4", memory: "8Gi" }
          volumeMounts:
            - { name: config, mountPath: /etc/pgdog }
            - { name: users, mountPath: /etc/secrets/pgdog }
      volumes:
        - { name: config, configMap: { name: pgdog-lt-config } }
        - { name: users, secret: { secretName: pgdog-lt-users } }
---
apiVersion: v1
kind: Service
metadata: { name: pgdog-lt-default, namespace: pgdog-lt }
spec:
  selector: { app: pgdog-lt-default }
  ports: [ { port: 6432, targetPort: 6432 } ]
---
# pgdog with _RJEM_MALLOC_CONF=background_thread — RSS returns to baseline
apiVersion: apps/v1
kind: Deployment
metadata: { name: pgdog-lt-bg, namespace: pgdog-lt }
spec:
  replicas: 1
  selector: { matchLabels: { app: pgdog-lt-bg } }
  template:
    metadata: { labels: { app: pgdog-lt-bg } }
    spec:
      containers:
        - name: pgdog
          image: ghcr.io/pgdogdev/pgdog:0.1.48
          command: ["/usr/local/bin/pgdog"]
          args: ["--config","/etc/pgdog/pgdog.toml","--users","/etc/secrets/pgdog/users.toml"]
          env:
            - { name: _RJEM_MALLOC_CONF, value: "background_thread:true" }
          ports: [ { containerPort: 6432 }, { containerPort: 9090 } ]
          resources:
            requests: { cpu: "500m", memory: "200Mi" }
            limits: { cpu: "4", memory: "8Gi" }
          volumeMounts:
            - { name: config, mountPath: /etc/pgdog }
            - { name: users, mountPath: /etc/secrets/pgdog }
      volumes:
        - { name: config, configMap: { name: pgdog-lt-config } }
        - { name: users, secret: { secretName: pgdog-lt-users } }
---
apiVersion: v1
kind: Service
metadata: { name: pgdog-lt-bg, namespace: pgdog-lt }
spec:
  selector: { app: pgdog-lt-bg }
  ports: [ { port: 6432, targetPort: 6432 } ]
---
# loadgen (has pgbench/psql from the postgres image)
apiVersion: apps/v1
kind: Deployment
metadata: { name: loadgen, namespace: pgdog-lt }
spec:
  replicas: 1
  selector: { matchLabels: { app: loadgen } }
  template:
    metadata: { labels: { app: loadgen } }
    spec:
      containers:
        - name: loadgen
          image: postgres:18
          command: ["sleep","infinity"]
          resources:
            requests: { cpu: "500m", memory: "512Mi" }
            limits: { cpu: "6", memory: "4Gi" }