Reproducer: proxy_v2 ctx->settings/ctx->pings not reset on upstream reinit

proxy_v2_settings_reinit.py

#!/usr/bin/env python3
"""
Reproducer for proxy_v2 ctx->settings / ctx->pings not reset on reinit.

Demonstrates that the DoS counters accumulate across upstream retries,
causing a healthy second upstream to be falsely rejected with
"upstream sent too many settings frames".

Usage:
    1. Build nginx with --with-http_v2_module
    2. Run:  python3 t/proxy_v2_settings_reinit.py <nginx_binary>

The script manages nginx lifecycle and both mock upstreams automatically.
"""

import os
import sys
import socket
import struct
import signal
import subprocess
import tempfile
import textwrap
import threading
import time


BAD_PORT = 10081
GOOD_PORT = 10082
NGINX_PORT = 10080

# Number of SETTINGS frames the bad upstream sends in one burst.
# Must reach 1000 so that one more from the good upstream exceeds the limit.
SETTINGS_COUNT = 1000


def h2_frame(ftype, flags=0, stream_id=0, payload=b''):
    """Build a raw HTTP/2 frame."""
    length = len(payload)
    header = struct.pack('>I', length)[1:] + struct.pack(
        '>BBL', ftype, flags, stream_id & 0x7FFFFFFF)
    return header + payload


def bad_upstream():
    """
    Accepts one HTTP/2 connection, sends SETTINGS_COUNT empty SETTINGS
    frames in a single write (so nginx processes them all in one read
    event with ctx->free == NULL the whole time), then closes.
    """
    srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    srv.bind(('127.0.0.1', BAD_PORT))
    srv.listen(1)
    srv.settimeout(5)

    try:
        conn, _ = srv.accept()
    except socket.timeout:
        srv.close()
        return

    # Read nginx's HTTP/2 client preface + SETTINGS + possible HEADERS
    try:
        conn.recv(16384)
    except Exception:
        pass

    # Send our SETTINGS preface + (SETTINGS_COUNT - 1) more, all at once
    settings_frame = h2_frame(0x04)  # type=SETTINGS, empty payload
    buf = settings_frame * SETTINGS_COUNT
    try:
        conn.sendall(buf)
    except Exception:
        pass

    # Small delay so nginx reads the burst before we close
    time.sleep(0.05)
    conn.close()
    srv.close()


def good_upstream():
    """
    Accepts one HTTP/2 connection, does a proper handshake, and responds
    with 200 OK on stream 1.
    """
    srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    srv.bind(('127.0.0.1', GOOD_PORT))
    srv.listen(1)
    srv.settimeout(5)

    try:
        conn, _ = srv.accept()
    except socket.timeout:
        srv.close()
        return

    conn.settimeout(3)

    # Read nginx client preface (24-byte magic + SETTINGS + HEADERS)
    try:
        data = conn.recv(16384)
    except Exception:
        conn.close()
        srv.close()
        return

    # Send server SETTINGS (empty)
    buf = h2_frame(0x04)

    # Send SETTINGS ACK (type=0x04, flags=0x01)
    buf += h2_frame(0x04, flags=0x01)

    # Send HEADERS response: :status 200, END_STREAM | END_HEADERS
    # HPACK indexed representation: 0x88 = index 8 = ":status: 200"
    headers_payload = bytes([0x88])
    buf += h2_frame(0x01, flags=0x05, stream_id=1, payload=headers_payload)

    try:
        conn.sendall(buf)
    except Exception:
        pass

    time.sleep(0.5)
    conn.close()
    srv.close()


def write_nginx_conf(tmpdir):
    """Write a minimal nginx.conf for the test."""
    conf_path = os.path.join(tmpdir, 'nginx.conf')
    logs_dir = os.path.join(tmpdir, 'logs')
    os.makedirs(logs_dir, exist_ok=True)

    conf = textwrap.dedent(f"""\
        daemon off;
        master_process off;
        worker_processes 1;

        pid {tmpdir}/nginx.pid;
        error_log {tmpdir}/logs/error.log debug;

        events {{
            worker_connections 64;
        }}

        http {{
            access_log {tmpdir}/logs/access.log;

            upstream backend {{
                server 127.0.0.1:{BAD_PORT};
                server 127.0.0.1:{GOOD_PORT};
            }}

            server {{
                listen {NGINX_PORT};

                location / {{
                    proxy_pass http://backend;
                    proxy_http_version 2;
                    proxy_next_upstream error timeout;
                    proxy_connect_timeout 2s;
                    proxy_read_timeout 2s;
                }}
            }}
        }}
    """)

    with open(conf_path, 'w') as f:
        f.write(conf)

    return conf_path


def wait_for_port(port, timeout=3):
    """Wait until a TCP port accepts connections."""
    deadline = time.monotonic() + timeout
    while time.monotonic() < deadline:
        try:
            s = socket.create_connection(('127.0.0.1', port), timeout=0.2)
            s.close()
            return True
        except OSError:
            time.sleep(0.05)
    return False


def main():
    if len(sys.argv) < 2:
        print(f'Usage: {sys.argv[0]} <path-to-nginx-binary>')
        sys.exit(1)

    nginx_bin = os.path.abspath(sys.argv[1])
    if not os.path.isfile(nginx_bin):
        print(f'Error: nginx binary not found at {nginx_bin}')
        sys.exit(1)

    tmpdir = tempfile.mkdtemp(prefix='nginx_test_')
    conf_path = write_nginx_conf(tmpdir)
    error_log = os.path.join(tmpdir, 'logs', 'error.log')

    print(f'[*] temp dir: {tmpdir}')
    print(f'[*] starting bad upstream on :{BAD_PORT} '
          f'(will send {SETTINGS_COUNT} SETTINGS frames)')
    print(f'[*] starting good upstream on :{GOOD_PORT}')

    bad_thread = threading.Thread(target=bad_upstream, daemon=True)
    good_thread = threading.Thread(target=good_upstream, daemon=True)
    bad_thread.start()
    good_thread.start()

    # Wait for upstreams to be ready
    wait_for_port(BAD_PORT)
    wait_for_port(GOOD_PORT)

    print(f'[*] starting nginx on :{NGINX_PORT}')
    nginx_proc = subprocess.Popen(
        [nginx_bin, '-c', conf_path],
        stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    if not wait_for_port(NGINX_PORT, timeout=3):
        print('[!] nginx failed to start')
        nginx_proc.kill()
        sys.exit(1)

    # Send a request to nginx (plain HTTP/1.1 on the client side)
    print('[*] sending request to nginx...')
    time.sleep(0.2)

    try:
        s = socket.create_connection(('127.0.0.1', NGINX_PORT), timeout=5)
        s.sendall(b'GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n')

        response = b''
        while True:
            chunk = s.recv(4096)
            if not chunk:
                break
            response += chunk
        s.close()
    except Exception as e:
        response = f'connection error: {e}'.encode()

    # Wait for threads
    bad_thread.join(timeout=3)
    good_thread.join(timeout=3)

    # Stop nginx
    try:
        nginx_proc.send_signal(signal.SIGTERM)
        nginx_proc.wait(timeout=3)
    except Exception:
        nginx_proc.kill()

    # Analyze results
    print()
    status_line = response.split(b'\r\n')[0].decode(errors='replace') \
        if response else '(empty response)'
    print(f'[*] response: {status_line}')

    # Check error log for the telltale message
    log_content = ''
    if os.path.isfile(error_log):
        with open(error_log) as f:
            log_content = f.read()

    too_many = 'upstream sent too many settings frames' in log_content

    print()
    if too_many:
        print('[BUG CONFIRMED] nginx error log contains:')
        print('  "upstream sent too many settings frames"')
        print()
        print('  The counter accumulated from upstream 1 (which sent '
              f'{SETTINGS_COUNT} SETTINGS)')
        print('  was NOT reset on reinit, so upstream 2\'s single normal')
        print('  SETTINGS frame pushed it past the 1000 limit.')
        sys.exit(1)
    elif 'HTTP/1.1 502' in status_line or 'HTTP/1.1 5' in status_line:
        print('[LIKELY BUG] got 502/5xx — check the error log:')
        print(f'  {error_log}')
        # Print relevant error lines
        for line in log_content.splitlines():
            if 'error' in line.lower() or 'too many' in line.lower():
                print(f'  {line.strip()}')
        sys.exit(1)
    elif 'HTTP/1.1 200' in status_line:
        print('[PASS] got 200 — bug is fixed (counter was reset on reinit)')
        sys.exit(0)
    else:
        print(f'[INCONCLUSIVE] unexpected response — check logs at:')
        print(f'  {error_log}')
        sys.exit(2)


if __name__ == '__main__':
    main()

Reproducer script for nginx/nginx#1199.

This script was generated with the assistance of Claude Opus (Anthropic) to validate the bug: ctx->pings and ctx->settings DoS counters in ngx_http_proxy_v2_reinit_request() are not reset on upstream retry, causing a healthy second upstream to be falsely rejected with "upstream sent too many settings frames".

Usage:

python3 proxy_v2_settings_reinit.py ./objs/nginx

The script spins up two mock HTTP/2 upstreams and an nginx instance automatically. The first upstream sends 1000 SETTINGS frames in a burst then closes; the second is a normal HTTP/2 server. Without the fix, the second upstream's single SETTINGS frame pushes the accumulated counter past the 1000 limit, triggering a false 502.