Reproducers for seastar issue #1784 / PR #1788: io_submit EAGAIN abort in aio_general_context::flush()

README.md

Reproducers for seastar issue #1784 / PR #1788

io_submit EAGAIN + abort on SEASTAR_ASSERT(clock::now() < retry_until) in aio_general_context::flush().

Two programs:

1. raw_aio_eagain.c — kernel-only demonstration (no seastar)

Shows the kernel mechanics directly: creates an aio context, reads the actual completion-ring capacity from the mmap'd ring header, then submits IOCB_CMD_POLL iocbs on idle socketpairs (which never complete, so each holds a ring slot forever) until io_submit returns EAGAIN, and finally retries for 1.5 s to show the EAGAIN never clears.

cc -O2 -o raw_aio_eagain raw_aio_eagain.c
ulimit -n 100000
./raw_aio_eagain 50       # or 1000, 10000 = seastar's default

Example output (kernel 6.17, 32 possible CPUs):

io_setup(50): kernel ring capacity (ring->nr) = 383
io_submit -> EAGAIN after 382 in-flight poll iocbs (ring capacity 383)
retrying for 1.5s (like aio_general_context::flush)...
  ...still EAGAIN after 7081409 attempts over 1.5s -- this is where seastar's
  SEASTAR_ASSERT(clock::now() < retry_until) would abort

Measured ring sizes (io_setup(nr) → max(nr, 4*num_possible_cpus) * 2 + 2, rounded up to page granularity, fs/aio.c ioctx_alloc/aio_setup_ring):

requested nr	ring capacity	EAGAIN at (in-flight iocbs)
50	383	382
1000	2047	2040
10000 (seastar default)	20095	19819

2. aio_eagain_repro.cc — seastar-level reproducer

Opens N idle loopback connections in one shard; each connection parks two never-completing POLLIN polls (client fd + server fd). Once the parked polls exhaust the ring, flush() spins on EAGAIN for 1 s and aborts.

clang++ -o aio_eagain_repro aio_eagain_repro.cc \
    $(pkg-config --cflags --libs /path/to/seastar/build/dev/seastar.pc)
ulimit -n 100000
./aio_eagain_repro --smp 1 --reactor-backend linux-aio \
    --max-networking-io-control-blocks 50 --conns 2000

Expected (crash while opening connection ~190 = (382 usable slots − 3 housekeeping polls) / 2 per connection):

opened 150 connections (302 parked polls)
src/core/reactor_backend.cc:462: size_t seastar::aio_general_context::flush():
    Assertion `clock::now() < retry_until` failed.
Aborting on shard 0 ...

With --max-networking-io-control-blocks 1000 it dies at ~1019 connections (2046 usable slots), and with the default 10000 it would take ~10k idle connections per shard — the Redpanda field-crash scenario.

aio_eagain_repro.cc

// Reproducer for https://github.com/scylladb/seastar/issues/1784
//
// The linux-aio reactor backend polls network fds by submitting
// IOCB_CMD_POLL iocbs to an aio context created with
// io_setup(max_networking_aio_io_control_blocks).  The kernel sizes the
// completion ring from that number (roughly 2x, with a floor of
// 4*num_possible_cpus, rounded up to page granularity) and every
// *in-flight* iocb reserves a completion-ring slot at io_submit time.
//
// A poll iocb for an idle connection never completes, so it holds its
// ring slot forever.  Once the number of in-flight polls reaches the
// ring capacity, io_submit() returns EAGAIN and no completion can ever
// free a slot => aio_general_context::flush() spins for 1s and then
// dies on SEASTAR_ASSERT(clock::now() < retry_until).
//
// This app opens many idle loopback connections (each contributes two
// never-completing POLLIN polls: client fd + server fd).  Run with a
// small ring:
//
//   ./aio_eagain_repro --smp 1 --reactor-backend linux-aio \
//       --max-networking-io-control-blocks 50 --conns 2000
//
#include <seastar/core/app-template.hh>
#include <seastar/core/coroutine.hh>
#include <seastar/core/future-util.hh>
#include <seastar/core/reactor.hh>
#include <seastar/core/seastar.hh>
#include <seastar/core/sleep.hh>
#include <seastar/net/api.hh>

#include <boost/program_options.hpp>

#include <iostream>
#include <memory>
#include <vector>

using namespace seastar;
using namespace std::chrono_literals;

struct conn {
    connected_socket sock;
    input_stream<char> in;
    explicit conn(connected_socket s) : sock(std::move(s)), in(sock.input()) {}
};

static std::vector<std::unique_ptr<conn>> connections;

// Issue a read that will never complete (peer never writes), parking a
// POLLIN poll iocb in the aio ring for the lifetime of the connection.
static void park_read(conn& c) {
    (void)c.in.read().then_wrapped([] (future<temporary_buffer<char>> f) {
        f.ignore_ready_future();
    });
}

static future<> accept_loop(server_socket& listener, unsigned n) {
    for (unsigned i = 0; i < n; ++i) {
        auto ar = co_await listener.accept();
        auto c = std::make_unique<conn>(std::move(ar.connection));
        park_read(*c);
        connections.push_back(std::move(c));
    }
}

int main(int ac, char** av) {
    app_template app;
    namespace bpo = boost::program_options;
    app.add_options()
        ("conns", bpo::value<unsigned>()->default_value(2000), "number of idle connections to open");
    return app.run(ac, av, [&app] () -> future<> {
        const auto nconns = app.configuration()["conns"].as<unsigned>();
        const uint16_t port = 23456;

        listen_options lo;
        lo.reuse_address = true;
        auto listener = listen(make_ipv4_address({port}), lo);
        auto accept_done = accept_loop(listener, nconns);

        for (unsigned i = 0; i < nconns; ++i) {
            auto s = co_await connect(make_ipv4_address({port}));
            auto c = std::make_unique<conn>(std::move(s));
            park_read(*c);
            connections.push_back(std::move(c));
            if (i % 50 == 0) {
                std::cout << "opened " << i << " connections ("
                          << connections.size() << " parked polls)" << std::endl;
            }
            // Pace connection creation so the reactor flushes the iocb
            // queue regularly: we want to hit the EAGAIN retry loop in
            // flush(), not the queue-full assert in queue().
            if (i % 5 == 4) {
                co_await sleep(1ms);
            }
        }
        co_await std::move(accept_done);
        std::cout << "all " << nconns << " connections open; idling (no crash?)" << std::endl;
        co_await sleep(10s);
        connections.clear();
    });
}

raw_aio_eagain.c

/*
 * Raw demonstration of the io_submit EAGAIN behavior behind seastar
 * issue #1784, with no seastar involved.
 *
 * io_setup(nr) sizes the completion ring as roughly:
 *   ring = round_up_to_pages(2 * max(nr, 4 * num_possible_cpus) + 2)
 * and every in-flight iocb reserves a ring slot at io_submit() time
 * (fs/aio.c get_reqs_available).  Slots are only released when
 * userspace consumes events (ring head advances).  An IOCB_CMD_POLL
 * on an idle fd never completes, so its slot is held forever: after
 * ~ring-capacity such submissions io_submit() returns EAGAIN and will
 * keep returning EAGAIN no matter how long you retry.
 *
 * Usage: ./raw_aio_eagain [nr_events]   (default 50)
 */
#define _GNU_SOURCE
#include <errno.h>
#include <linux/aio_abi.h>
#include <poll.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/syscall.h>
#include <time.h>
#include <unistd.h>

static int io_setup(unsigned nr, aio_context_t *ctx) {
    return syscall(SYS_io_setup, nr, ctx);
}
static int io_submit(aio_context_t ctx, long n, struct iocb **iocbs) {
    return syscall(SYS_io_submit, ctx, n, iocbs);
}

/* layout of the mmap'd ring header, from fs/aio.c */
struct aio_ring {
    unsigned id, nr, head, tail;
    unsigned magic, compat_features, incompat_features, header_length;
};

int main(int argc, char **argv) {
    unsigned nr = argc > 1 ? atoi(argv[1]) : 50;
    aio_context_t ctx = 0;
    if (io_setup(nr, &ctx) < 0) {
        perror("io_setup");
        return 1;
    }
    struct aio_ring *ring = (struct aio_ring *)ctx;
    printf("io_setup(%u): kernel ring capacity (ring->nr) = %u\n", nr, ring->nr);

    enum { MAX = 100000 };
    for (unsigned i = 0; i < MAX; i++) {
        int sv[2];
        if (socketpair(AF_UNIX, SOCK_STREAM, 0, sv) < 0) {
            perror("socketpair");
            return 1;
        }
        /* poll for POLLIN on an fd nobody will ever write to */
        struct iocb *cb = calloc(1, sizeof(*cb));
        cb->aio_lio_opcode = IOCB_CMD_POLL;
        cb->aio_fildes = sv[0];
        cb->aio_buf = POLLIN;
        struct iocb *list[1] = { cb };
        int r = io_submit(ctx, 1, list);
        if (r < 0) {
            if (errno == EAGAIN) {
                printf("io_submit -> EAGAIN after %u in-flight poll iocbs "
                       "(ring capacity %u)\n", i, ring->nr);
                printf("retrying for 1.5s (like aio_general_context::flush)...\n");
                struct timespec start, now;
                clock_gettime(CLOCK_MONOTONIC, &start);
                unsigned long attempts = 0;
                for (;;) {
                    r = io_submit(ctx, 1, list);
                    attempts++;
                    if (r == 1) {
                        printf("  ...unexpectedly succeeded after %lu attempts\n", attempts);
                        return 0;
                    }
                    clock_gettime(CLOCK_MONOTONIC, &now);
                    double el = (now.tv_sec - start.tv_sec) + (now.tv_nsec - start.tv_nsec) / 1e9;
                    if (el > 1.5) {
                        printf("  ...still EAGAIN after %lu attempts over %.1fs -- "
                               "this is where seastar's SEASTAR_ASSERT(clock::now() < retry_until) "
                               "would abort\n", attempts, el);
                        return 0;
                    }
                }
            }
            perror("io_submit");
            return 1;
        }
    }
    printf("submitted %u polls without EAGAIN?!\n", MAX);
    return 0;
}