Running Mastodon on Kubernetes

000-provision.sh

#!/usr/bin/env bash

# Install nginx ingress controller
# NOTE: This uses DigitalOcean. If you use another Kubernetes provider,
# substitute the appropriate command from here: https://kubernetes.github.io/ingress-nginx/deploy/#cloud-deployments
kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.5.1/deploy/static/provider/do/deploy.yaml

# FOR DIGITALOCEAN DEPLOYMENTS:
export LB_HOSTNAME=example.com # Make this the DNS name that will point to your LB
export LB_NAME=my-lb # Give this a useful name to identify it on the DigitalOcean control panel
echo '{"metadata":{"annotations":{"service.beta.kubernetes.io/do-loadbalancer-hostname":"$LB_HOSTNAME","service.beta.kubernetes.io/do-loadbalancer-name":"$LB_NAME"}}}' |
  envsubst |
  awk "{ print \"'\" \$1 \"'\" }" |
  xargs kubectl patch svc -n ingress-nginx ingress-nginx-controller -p

# Install CertManager
kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.10.0/cert-manager.yaml

# Install RailsApp CRD+Operator
kubectl apply -f https://raw.githubusercontent.com/jgaskins/rails_app_operator/main/k8s/crd-rails-app.yaml
kubectl apply -f https://raw.githubusercontent.com/jgaskins/rails_app_operator/main/k8s/operator.yaml

# Install CloudNative Postgres operator
kubectl apply -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-1.18/releases/cnpg-1.18.0.yaml

# Install Elasticsearch Operator (optional)
# kubectl apply -f https://download.elastic.co/downloads/eck/2.5.0/operator.yaml

# Provision a ClusterIssuer for CertManager
cat <<EOF | kubectl apply -f -
---
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: letsencrypt
spec:
  acme:
    server: https://acme-v02.api.letsencrypt.org/directory
    privateKeySecretRef:
      name: issuer-letsencrypt-account-key
    solvers:
    - http01:
       ingress:
         class: nginx
EOF

all-the-things.yaml

# Look through the `env` entries for the `RailsApp` and also the `Secret` at the bottom.
# You will need to change those to configure things.

---
apiVersion: v1
kind: Namespace
metadata:
  name: mastodon
---
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
  name: postgres
  namespace: mastodon
spec:
  instances: 1
  storage:
    size: 10Gi
---
apiVersion: v1
kind: Service
metadata:
  name: redis
  namespace: mastodon
spec:
  selector:
    app: redis
  ports:
  - port: 6379
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis
  namespace: mastodon
spec:
  selector:
    matchLabels:
      app: redis
  serviceName: redis
  template:
    metadata:
      labels:
        app: redis
    spec:
      containers:
      # TODO: Add memory resource limits
      - name: redis
        image: redis
        ports:
        - containerPort: 6379
          name: redis
        volumeMounts:
        - name: data
          mountPath: /data
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes: [ReadWriteOnce]
      resources:
        requests:
          storage: 1Gi
---
apiVersion: jgaskins.dev/v1beta1
kind: RailsApp
metadata:
  name: mastodon
  namespace: mastodon
spec:
  image: tootsuite/mastodon:v4.0.2
  # Uncomment this if using a mutable container image tag
  # image_pull_policy: Always
  env:
    # Rails-specific stuff
    - name: RAILS_ENV
      value: production
    - name: DB_NAME
      value: app
    - name: DB_HOST
      value: postgres-rw
    - name: DB_USER
      valueFrom:
        secretKeyRef:
          name: postgres-superuser
          key: username
    - name: DB_PASS
      valueFrom:
        secretKeyRef:
          name: postgres-superuser
          key: password
    - name: REDIS_URL
      value: redis://redis:6379/
    - name: SECRET_KEY_BASE
      valueFrom:
        secretKeyRef:
          name: mastodon
          key: secret_key_base
    - name: OTP_SECRET
      valueFrom:
        secretKeyRef:
          name: mastodon
          key: otp_secret
    - name: RAILS_SERVE_STATIC_FILES
      value: "true"
    - name: RAILS_LOG_TO_STDOUT
      value: "true"
    - name: RAILS_LOG_LEVEL
      value: "info"

    # Mastodon-specific stuff
    - name: LOCAL_DOMAIN
      value: 
    - name: STREAMING_API_BASE_URL
      value: 

    # Email
    - name: SMTP_SERVER
      value: 
    - name: SMTP_FROM_ADDRESS
      value: 
    - name: SMTP_LOGIN
      valueFrom:
        secretKeyRef:
          name: mastodon
          key: smtp_login
    - name: SMTP_PASSWORD
      valueFrom:
        secretKeyRef:
          name: mastodon
          key: smtp_password

    # File storage
    - name: S3_ALIAS_HOST
      value: 
    - name: AWS_ACCESS_KEY_ID
      valueFrom:
        secretKeyRef:
          name: mastodon
          key: aws_access_key_id
    - name: AWS_SECRET_ACCESS_KEY
      valueFrom:
        secretKeyRef:
          name: mastodon
          key: aws_secret_access_key
    - name: S3_ENABLED
      value: "true"
    - name: S3_BUCKET
      value: 
    - name: S3_REGION
      value: 
    - name: S3_ENDPOINT
      value: 
    - name: S3_OVERRIDE_PATH_STYLE
      value: "true"

    # OpenTelemetry
    # Requires image: jgaskins/mastodon:rub-some-otel-on-it
    - name: OTEL_EXPORTER_OTLP_ENDPOINT
      value: http://otel-collector:4318

    # Elasticsearch
    # - name: ES_ENABLED
    #   value: "true"
    # - name: ES_HOST
    #   value: "elasticsearch-es-http"
    # - name: ES_USER
    #   value: "elastic"
    # - name: ES_PASS
    #   valueFrom:
    #     secretKeyRef:
    #       name: elasticsearch-es-elastic-user
    #       key: elastic

  entrypoints:
    - name: web
      command: [bundle, exec, rails, server]
      domain: 
      port: 3000
      replicas: 1 # Defaults to 1
      env:
        - name: WEB_CONCURRENCY
          value: "0"
        - name: MAX_THREADS
          value: "15"
      health_check:
        path: "/health"
        start_after: 10
        run_every: 5
        failure_threshold: 36 # 3 minutes
    - name: streaming
      command: [node, ./streaming]
      replicas: 1 # Defaults to 1
      domain: 
      port: 4000
      env:
        - name: NODE_ENV
          value: production
        - name: STREAMING_CLUSTER_NUM
          value: "1"
      health_check:
        path: "/api/v1/streaming/health"
    - name: sidekiq
      command: [bundle, exec, sidekiq]
      replicas: 1 # Defaults to 1
      env:
        - name: RAILS_MAX_THREADS
          value: "15"

  before_create:
    command: ["/bin/bash", "-c", "bundle exec rails db:setup && bin/tootctl accounts create $ADMIN_USERNAME --email $ADMIN_EMAIL --confirmed --role Admin && rails runner \"User.where(email: \"$ADMIN_EMAIL\").update_all(encrypted_password: BCrypt::Password.create(\\\"$ADMIN_PASSWORD\\\")\""]
    env:
      - name: OTEL_SERVICE_NAME
        value: "db:setup"
      - name: ADMIN_USERNAME
        valueFrom:
          secretKeyRef:
            name: mastodon
            key: admin_username
      - name: ADMIN_EMAIL
        valueFrom:
          secretKeyRef:
            name: mastodon
            key: admin_email
      - name: ADMIN_PASSWORD
        valueFrom:
          secretKeyRef:
            name: mastodon
            key: admin_password

  before_update:
    command: [bundle, exec, rails, db:migrate]
    env:
      - name: OTEL_SERVICE_NAME
        value: "db:migrate"

---
#### Uncomment to run Elasticsearch in your cluster
# apiVersion: elasticsearch.k8s.elastic.co/v1
# kind: Elasticsearch
# metadata:
#   name: elasticsearch
#   namespace: mastodon
# spec:
#   version: 8.5.0
#   http:
#     tls:
#       selfSignedCertificate:
#         disabled: true
#   nodeSets:
#   - name: default
#     count: 1
#     config:
#       node.store.allow_mmap: false

---
#### Needs this PR to be useful: https://github.com/mastodon/mastodon/pull/20338
#### You can use the container image: jgaskins/mastodon:rub-some-otel-on-it
# apiVersion: opentelemetry.io/v1alpha1
# kind: OpenTelemetryCollector
# metadata:
#   name: otel
#   namespace: mastodon
# spec:
#   config: |
#     receivers:
#       otlp:
#         protocols:
#           grpc:
#           http:
# 
#     processors:
#       batch:
#       probabilistic_sampler:
#         sampling_percentage: 20
# 
#     exporters:
#       otlp:
#         # Get this from your OTel provider
#         endpoint:
#         headers:
# 
#     service:
#       pipelines:
#         traces:
#           receivers: [otlp]
#           processors: [probabilistic_sampler, batch]
#           exporters: [otlp]
# ---
apiVersion: v1
kind: Secret
metadata:
  name: mastodon
  namespace: mastodon
stringData:
  aws_access_key_id: 
  aws_secret_access_key:
  secret_key_base: # Any long random string
  otp_secret: # Any long random string
  smtp_login: 
  smtp_password: 
  admin_username: 
  admin_email: 
  admin_password: