gistfile1.txt

spec: &spec
  x-sub-request-filters:
    - type: default
      name: http
      options:
        allow:
          - pattern: /^https?:\/\//
            forward_headers:
              user-agent: true
  title: The Change Propagation root
  paths:
    /{api:sys}/links:
      x-modules:
        - path: src/sys/dep_updates.js
          options:
            templates:
              mw_api:
                uri: http://api-rw.discovery.wmnet/w/api.php
                headers:
                  host: '{{message.meta.domain}}'
                body:
                  formatversion: 2
    /{api:sys}/queue:
      x-modules:
        - path: src/sys/kafka.js
          options:
            metadata_broker_list: kafka1001.eqiad.wmnet:9092,kafka1002.eqiad.wmnet:9092,kafka1003.eqiad.wmnet:9092
            dc_name: eqiad
            concurrency: 50
            startup_delay: 60000
            test_mode: true
            sample:
              hash_template: '{{message.meta.uri}}'
              rate: 0.3
            templates:
              page_edit:
                topic: mediawiki.revision-create
                retry_on:
                  status:
                    - '5xx'
                    - 404 # Sometimes occasional 404s happen because of the mysql replication lag, so retry
                match:
                  rev_content_changed: true
                match_not:
                  - meta:
                      domain: /\.wikidata\.org$/
                    page_namespace: 0
                  - meta:
                      domain: /\.wikidata\.org$/
                    page_namespace: 120
                exec:
                  - method: get
                    uri: 'http://localhost:7231/{{message.meta.domain}}/v1/page/html/{message.page_title}/{{message.rev_id}}'
                    headers:
                      cache-control: no-cache
                      x-restbase-parentrevision: '{{message.rev_parent_id}}'
                      if-unmodified-since: '{{date(message.meta.dt)}}'
                    query:
                      redirect: false

# Number of worker processes to spawn.
# Set to 0 to run everything in a single process without clustering.
# Use 'ncpu' to run as many workers as there are CPU units
num_workers: ncpu

# Log error messages and gracefully restart a worker if v8 reports that it
# uses more heap (note: not RSS) than this many mb.
worker_heap_limit_mb: 750

# The maximum interval in ms that can pass between two beat messages
# sent by each worker to the master before it is killed
worker_heartbeat_timeout: 15000

# Logger info
logging:
  level: warn
  name: changeprop-dev
  streams:
    - level: info
      path: /srv/log/changeprop/main.log
      type: file

# Statsd metrics reporter
metrics:
  name: changeprop-dev
  host: statsd.eqiad.wmnet
  port: 8125
  type: statsd

services:
  - name: changeprop
    # a relative path or the name of an npm package, if different from name
    module: hyperswitch
    # per-service config
    conf:
      cors: "*"
      port: 7272
      # interface: localhost # uncomment to only listen on localhost
      # URL of the outbound proxy to use (complete with protocol)
      proxy: http://url-downloader.eqiad.wikimedia.org:8080
      # the list of domains for which not to use the proxy defined above
      # no_proxy_list:
      #   - domain1.com
      #   - domain2.org
      # the list of incoming request headers that can be logged; if left empty,
      # the following headers are allowed: cache-control, content-length,
      # content-type, if-match, user-agent, x-request-id
      # log_header_whitelist:
      #   - cache-control
      #   - content-length
      #   - content-type
      #   - if-match
      #   - user-agent
      #   - x-request-id
      user_agent: ChangePropagation/WMF
      spec: *spec