ngtcp2 と nghttp3 で HTTP/3 クライアント (OpenSSL)

01-README.md

ngtcp2 と nghttp3 で HTTP/3 クライアント (OpenSSL)

OpenSSL、nghttp3、ngtcp2 のインストール

Debian 12 (bookworm)

export PREFIX=$HOME/.local
export PATH="$PREFIX/bin:$PATH"
export PKG_CONFIG_PATH="$PREFIX/lib64/pkgconfig:$PREFIX/lib/pkgconfig:$PKG_CONFIG_PATH"
export LD_LIBRARY_PATH="$PREFIX/lib64:$PREFIX/lib:$LD_LIBRARY_PATH"

OpenSSL 3.6

cd openssl
make clean
./config --prefix=$HOME/.local --openssldir=$HOME/.local
make -j$(nproc)
make install

nghttp3

cd nghttp3
git submodule update --init
autoreconf -i
./configure --prefix=$PREFIX
make -j$(nproc)
make install

ngtcp2

cd ngtcp2
git submodule update --init
autoreconf -i
./configure --prefix=$PREFIX
make -j$(nproc)
make install

ビルド

export PKG_CONFIG_PATH="$PREFIX/lib64/pkgconfig:$PREFIX/lib/pkgconfig:$PKG_CONFIG_PATH"
export LD_LIBRARY_PATH="$PREFIX/lib64:$PREFIX/lib:$LD_LIBRARY_PATH"

pkg-config --modversion libngtcp2 libngtcp2_crypto_ossl libnghttp3 openssl

cc -Wall -Wextra -O2 minimal_h3_client_openssl.c -o minimal_h3_client_openssl \
    $(pkg-config --cflags --libs libngtcp2 libngtcp2_crypto_ossl libnghttp3 openssl)

02-h3_client_openssl.c

#include <arpa/inet.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <netdb.h>
#include <openssl/err.h>
#include <openssl/rand.h>
#include <openssl/ssl.h>
#include <poll.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <time.h>
#include <unistd.h>

#include <nghttp3/nghttp3.h>
#include <ngtcp2/ngtcp2.h>
#include <ngtcp2/ngtcp2_crypto.h>
#if __has_include(<ngtcp2/ngtcp2_crypto_ossl.h>)
#  include <ngtcp2/ngtcp2_crypto_ossl.h>
#  define NGTCP2_OPENSSL_BACKEND_OSSL 1
#elif __has_include(<ngtcp2/ngtcp2_crypto_quictls.h>)
#  include <ngtcp2/ngtcp2_crypto_quictls.h>
#  define NGTCP2_OPENSSL_BACKEND_QUICTLS 1
#else
#  error "Need ngtcp2 OpenSSL backend header (ngtcp2_crypto_ossl.h or ngtcp2_crypto_quictls.h)"
#endif

#define REMOTE_HOST "nghttp2.org"
#define REMOTE_PORT "443"
#define REQ_PATH "/httpbin/get"

#define RXBUF_SIZE 65536
#define TXBUF_SIZE 2048

typedef struct client {
  int fd;

  struct sockaddr_storage local_addr;
  socklen_t local_addrlen;
  struct sockaddr_storage remote_addr;
  socklen_t remote_addrlen;
  ngtcp2_path_storage ps;

  ngtcp2_conn *qconn;
  nghttp3_conn *h3conn;

  SSL_CTX *ssl_ctx;
  SSL *ssl;
  ngtcp2_crypto_ossl_ctx *ossl_ctx;
  ngtcp2_crypto_conn_ref conn_ref;

  uint8_t rxbuf[RXBUF_SIZE];
  uint8_t txbuf[TXBUF_SIZE];

  int handshake_completed;
  int h3_streams_bound;
  int request_submitted;
  int response_complete;
  int64_t req_stream_id;
  ngtcp2_tstamp last_ts;

  int had_error;
  uint64_t tx_pkt_count;
  uint64_t rx_pkt_count;
} client;

static ngtcp2_tstamp timestamp_now_raw(void) {
  struct timespec ts;
  clock_gettime(CLOCK_MONOTONIC, &ts);
  return (ngtcp2_tstamp)ts.tv_sec * NGTCP2_SECONDS + (ngtcp2_tstamp)ts.tv_nsec;
}

static ngtcp2_tstamp timestamp_now(client *c) {
  ngtcp2_tstamp ts = timestamp_now_raw();
  if (ts <= c->last_ts) {
    ts = c->last_ts + 1;
  }
  c->last_ts = ts;
  return ts;
}

static void fail(client *c, const char *where) {
  if (!c->had_error) {
    fprintf(stderr, "error: %s\n", where);
  }
  c->had_error = 1;
}

static const char *ccerr_type_name(ngtcp2_ccerr_type t) {
  switch (t) {
  case NGTCP2_CCERR_TYPE_TRANSPORT:
    return "transport";
  case NGTCP2_CCERR_TYPE_APPLICATION:
    return "application";
  case NGTCP2_CCERR_TYPE_VERSION_NEGOTIATION:
    return "version_negotiation";
  case NGTCP2_CCERR_TYPE_IDLE_CLOSE:
    return "idle_close";
  case NGTCP2_CCERR_TYPE_DROP_CONN:
    return "drop_conn";
  case NGTCP2_CCERR_TYPE_RETRY:
    return "retry";
  default:
    return "unknown";
  }
}

static void log_connection_close_reason(client *c, const char *where) {
  const ngtcp2_ccerr *ccerr = ngtcp2_conn_get_ccerr(c->qconn);
  if (!ccerr) {
    fprintf(stderr, "%s: no ccerr\n", where);
    return;
  }

  fprintf(stderr,
          "%s: ccerr type=%s error_code=0x%llx frame_type=0x%llx reason=%.*s\n",
          where, ccerr_type_name(ccerr->type),
          (unsigned long long)ccerr->error_code,
          (unsigned long long)ccerr->frame_type, (int)ccerr->reasonlen,
          ccerr->reason ? (const char *)ccerr->reason : "");
}

static void log_ssl_errors(const char *where) {
  unsigned long err;
  int found = 0;
  while ((err = ERR_get_error()) != 0) {
    char buf[256];
    ERR_error_string_n(err, buf, sizeof(buf));
    fprintf(stderr, "%s: openssl: %s\n", where, buf);
    found = 1;
  }
  if (!found) {
    fprintf(stderr, "%s: openssl: no error in queue\n", where);
  }
}

static int file_exists(const char *path) {
  struct stat st;
  return stat(path, &st) == 0;
}

static ngtcp2_conn *get_conn(ngtcp2_crypto_conn_ref *ref) {
  client *c = (client *)ref->user_data;
  return c->qconn;
}

static void rand_cb(uint8_t *dest, size_t destlen,
                    const ngtcp2_rand_ctx *rand_ctx) {
  (void)rand_ctx;
  if (RAND_bytes(dest, (int)destlen) != 1) {
    memset(dest, 0, destlen);
  }
}

static int get_new_connection_id_cb(ngtcp2_conn *conn, ngtcp2_cid *cid,
                                    uint8_t *token, size_t cidlen,
                                    void *user_data) {
  (void)conn;
  (void)user_data;
  if (RAND_bytes(cid->data, (int)cidlen) != 1) {
    return NGTCP2_ERR_CALLBACK_FAILURE;
  }
  cid->datalen = cidlen;
  if (RAND_bytes(token, NGTCP2_STATELESS_RESET_TOKENLEN) != 1) {
    return NGTCP2_ERR_CALLBACK_FAILURE;
  }
  return 0;
}

static int q_client_initial_cb(ngtcp2_conn *conn, void *user_data) {
  int rv = ngtcp2_crypto_client_initial_cb(conn, user_data);
  if (rv != 0) {
    log_ssl_errors("client_initial");
  }
  return rv;
}

static int q_recv_crypto_data_cb(ngtcp2_conn *conn,
                                 ngtcp2_encryption_level encryption_level,
                                 uint64_t offset, const uint8_t *data,
                                 size_t datalen, void *user_data) {
  client *c = (client *)user_data;
  int rv = ngtcp2_crypto_recv_crypto_data_cb(conn, encryption_level, offset,
                                             data, datalen, user_data);
  if (rv != 0) {
    log_ssl_errors("recv_crypto_data");
    if (c && c->ssl) {
      long verr = SSL_get_verify_result(c->ssl);
      if (verr != X509_V_OK) {
        fprintf(stderr, "recv_crypto_data: verify_result=%ld (%s)\n", verr,
                X509_verify_cert_error_string(verr));
      }
    }
  }
  return rv;
}

static void extend_flow_control(client *c, int64_t stream_id, uint64_t n) {
  if (!n) {
    return;
  }
  if (ngtcp2_conn_extend_max_stream_offset(c->qconn, stream_id, n) != 0) {
    fail(c, "ngtcp2_conn_extend_max_stream_offset");
    return;
  }
  ngtcp2_conn_extend_max_offset(c->qconn, n);
}

static int h3_recv_data_cb(nghttp3_conn *conn, int64_t stream_id,
                           const uint8_t *data, size_t datalen,
                           void *conn_user_data, void *stream_user_data) {
  (void)conn;
  (void)stream_user_data;
  client *c = (client *)conn_user_data;

  if (datalen && fwrite(data, 1, datalen, stdout) != datalen) {
    return NGHTTP3_ERR_CALLBACK_FAILURE;
  }
  fflush(stdout);

  /* DATA frame payload bytes are not included in nghttp3_conn_read_stream's
     consumed value, so return credit explicitly. */
  extend_flow_control(c, stream_id, (uint64_t)datalen);
  return c->had_error ? NGHTTP3_ERR_CALLBACK_FAILURE : 0;
}

static int h3_deferred_consume_cb(nghttp3_conn *conn, int64_t stream_id,
                                  size_t consumed, void *conn_user_data,
                                  void *stream_user_data) {
  (void)conn;
  (void)stream_user_data;
  client *c = (client *)conn_user_data;
  extend_flow_control(c, stream_id, (uint64_t)consumed);
  return c->had_error ? NGHTTP3_ERR_CALLBACK_FAILURE : 0;
}

static int h3_recv_header_cb(nghttp3_conn *conn, int64_t stream_id,
                             int32_t token, nghttp3_rcbuf *name,
                             nghttp3_rcbuf *value, uint8_t flags,
                             void *conn_user_data, void *stream_user_data) {
  (void)conn;
  (void)stream_id;
  (void)token;
  (void)flags;
  (void)conn_user_data;
  (void)stream_user_data;

  nghttp3_vec n = nghttp3_rcbuf_get_buf(name);
  nghttp3_vec v = nghttp3_rcbuf_get_buf(value);

  if (n.len == 7 && memcmp(n.base, ":status", 7) == 0) {
    fprintf(stderr, "HTTP status: %.*s\n", (int)v.len, v.base);
  }
  return 0;
}

static int h3_end_stream_cb(nghttp3_conn *conn, int64_t stream_id,
                            void *conn_user_data, void *stream_user_data) {
  (void)conn;
  (void)stream_user_data;
  client *c = (client *)conn_user_data;
  if (stream_id == c->req_stream_id) {
    c->response_complete = 1;
  }
  return 0;
}

static int h3_stream_close_cb(nghttp3_conn *conn, int64_t stream_id,
                              uint64_t app_error_code, void *conn_user_data,
                              void *stream_user_data) {
  (void)conn;
  (void)stream_user_data;
  client *c = (client *)conn_user_data;
  if (stream_id == c->req_stream_id) {
    fprintf(stderr, "HTTP/3 stream closed (app_error=%" PRIu64 ")\n",
            app_error_code);
  }
  return 0;
}

static int h3_stop_sending_cb(nghttp3_conn *conn, int64_t stream_id,
                              uint64_t app_error_code, void *conn_user_data,
                              void *stream_user_data) {
  (void)conn;
  (void)stream_user_data;
  client *c = (client *)conn_user_data;
  if (ngtcp2_conn_shutdown_stream_read(c->qconn, 0, stream_id,
                                       app_error_code) < 0) {
    return NGHTTP3_ERR_CALLBACK_FAILURE;
  }
  return 0;
}

static int h3_reset_stream_cb(nghttp3_conn *conn, int64_t stream_id,
                              uint64_t app_error_code, void *conn_user_data,
                              void *stream_user_data) {
  (void)conn;
  (void)stream_user_data;
  client *c = (client *)conn_user_data;
  if (ngtcp2_conn_shutdown_stream_write(c->qconn, 0, stream_id,
                                        app_error_code) < 0) {
    return NGHTTP3_ERR_CALLBACK_FAILURE;
  }
  return 0;
}

static int q_recv_stream_data_cb(ngtcp2_conn *conn, uint32_t flags,
                                 int64_t stream_id, uint64_t offset,
                                 const uint8_t *data, size_t datalen,
                                 void *user_data, void *stream_user_data) {
  (void)conn;
  (void)offset;
  (void)stream_user_data;
  client *c = (client *)user_data;

  int fin = !!(flags & NGTCP2_STREAM_DATA_FLAG_FIN);
  nghttp3_ssize nconsumed =
    nghttp3_conn_read_stream(c->h3conn, stream_id, data, datalen, fin);
  if (nconsumed < 0) {
    fprintf(stderr, "nghttp3_conn_read_stream: %s\n", nghttp3_strerror((int)nconsumed));
    return NGTCP2_ERR_CALLBACK_FAILURE;
  }

  /* These consumed bytes are frame/QPACK/control bytes, not body bytes. */
  extend_flow_control(c, stream_id, (uint64_t)nconsumed);

  return c->had_error ? NGTCP2_ERR_CALLBACK_FAILURE : 0;
}

static int q_acked_stream_data_offset_cb(ngtcp2_conn *conn, int64_t stream_id,
                                         uint64_t offset, uint64_t datalen,
                                         void *user_data,
                                         void *stream_user_data) {
  (void)conn;
  (void)offset;
  (void)stream_user_data;
  client *c = (client *)user_data;
  if (nghttp3_conn_add_ack_offset(c->h3conn, stream_id, datalen) != 0) {
    return NGTCP2_ERR_CALLBACK_FAILURE;
  }
  return 0;
}

static int q_stream_close_cb(ngtcp2_conn *conn, uint32_t flags,
                             int64_t stream_id, uint64_t app_error_code,
                             void *user_data, void *stream_user_data) {
  (void)conn;
  (void)flags;
  (void)stream_user_data;
  client *c = (client *)user_data;
  if (nghttp3_conn_close_stream(c->h3conn, stream_id, app_error_code) != 0) {
    return NGTCP2_ERR_CALLBACK_FAILURE;
  }
  return 0;
}

static int q_handshake_completed_cb(ngtcp2_conn *conn, void *user_data) {
  (void)conn;
  client *c = (client *)user_data;
  c->handshake_completed = 1;
  fprintf(stderr, "QUIC handshake completed\n");
  return 0;
}

static int socket_connect(client *c) {
  struct addrinfo hints;
  struct addrinfo *res = NULL;
  struct addrinfo *rp;
  int rv;

  memset(&hints, 0, sizeof(hints));
  /* Prefer IPv4 in this minimal sample: UDP connect() on IPv6 can succeed
     even when path is not actually reachable in some environments. */
  hints.ai_family = AF_INET;
  hints.ai_socktype = SOCK_DGRAM;

  rv = getaddrinfo(REMOTE_HOST, REMOTE_PORT, &hints, &res);
  if (rv != 0) {
    fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(rv));
    return -1;
  }

  c->fd = -1;
  for (rp = res; rp; rp = rp->ai_next) {
    int fd = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol);
    if (fd < 0) {
      continue;
    }

    if (connect(fd, rp->ai_addr, rp->ai_addrlen) == 0) {
      c->fd = fd;
      memcpy(&c->remote_addr, rp->ai_addr, (size_t)rp->ai_addrlen);
      c->remote_addrlen = (socklen_t)rp->ai_addrlen;
      break;
    }

    close(fd);
  }
  freeaddrinfo(res);

  if (c->fd < 0) {
    perror("connect");
    return -1;
  }

  int fl = fcntl(c->fd, F_GETFL, 0);
  if (fl < 0 || fcntl(c->fd, F_SETFL, fl | O_NONBLOCK) < 0) {
    perror("fcntl(O_NONBLOCK)");
    return -1;
  }

  c->local_addrlen = sizeof(c->local_addr);
  if (getsockname(c->fd, (struct sockaddr *)&c->local_addr, &c->local_addrlen) !=
      0) {
    perror("getsockname");
    return -1;
  }

  ngtcp2_path_storage_init(&c->ps,
                           (struct sockaddr *)&c->local_addr, c->local_addrlen,
                           (struct sockaddr *)&c->remote_addr,
                           c->remote_addrlen,
                           NULL);

  fprintf(stderr, "connected UDP socket (IPv4) to " REMOTE_HOST ":" REMOTE_PORT "\n");

  return 0;
}

static int setup_tls(client *c) {
  static const uint8_t alpn[] = {2, 'h', '3'};

  if (ngtcp2_crypto_ossl_init() != 0) {
    fprintf(stderr, "ngtcp2_crypto_ossl_init failed\n");
    return -1;
  }

  if (OPENSSL_init_ssl(0, NULL) != 1) {
    fprintf(stderr, "OPENSSL_init_ssl failed\n");
    return -1;
  }

  c->ssl_ctx = SSL_CTX_new(TLS_client_method());
  if (!c->ssl_ctx) {
    fprintf(stderr, "SSL_CTX_new failed\n");
    return -1;
  }

  if (SSL_CTX_set_default_verify_paths(c->ssl_ctx) != 1) {
    fprintf(stderr,
            "SSL_CTX_set_default_verify_paths failed, trying explicit CA bundle paths\n");
  }
  if (file_exists("/etc/ssl/certs/ca-certificates.crt")) {
    (void)SSL_CTX_load_verify_locations(c->ssl_ctx,
                                        "/etc/ssl/certs/ca-certificates.crt",
                                        NULL);
  } else if (file_exists("/etc/pki/tls/certs/ca-bundle.crt")) {
    (void)SSL_CTX_load_verify_locations(c->ssl_ctx,
                                        "/etc/pki/tls/certs/ca-bundle.crt",
                                        NULL);
  }
  SSL_CTX_set_verify(c->ssl_ctx, SSL_VERIFY_PEER, NULL);
  SSL_CTX_set_min_proto_version(c->ssl_ctx, TLS1_3_VERSION);
  SSL_CTX_clear_options(c->ssl_ctx, SSL_OP_ENABLE_MIDDLEBOX_COMPAT);

#if defined(NGTCP2_OPENSSL_BACKEND_QUICTLS)
  if (ngtcp2_crypto_quictls_configure_client_context(c->ssl_ctx) != 0) {
    fprintf(stderr, "ngtcp2_crypto_quictls_configure_client_context failed\n");
    return -1;
  }
#endif

  c->ssl = SSL_new(c->ssl_ctx);
  if (!c->ssl) {
    fprintf(stderr, "SSL_new failed\n");
    return -1;
  }
  SSL_set_connect_state(c->ssl);

  if (SSL_set_tlsext_host_name(c->ssl, REMOTE_HOST) != 1) {
    fprintf(stderr, "SSL_set_tlsext_host_name failed\n");
    return -1;
  }
  if (SSL_set1_host(c->ssl, REMOTE_HOST) != 1) {
    fprintf(stderr, "SSL_set1_host failed\n");
    return -1;
  }
  if (SSL_set_alpn_protos(c->ssl, alpn, sizeof(alpn)) != 0) {
    fprintf(stderr, "SSL_set_alpn_protos failed\n");
    return -1;
  }

  c->conn_ref.get_conn = get_conn;
  c->conn_ref.user_data = c;
  SSL_set_app_data(c->ssl, &c->conn_ref);

#if defined(NGTCP2_OPENSSL_BACKEND_OSSL)
  if (ngtcp2_crypto_ossl_configure_client_session(c->ssl) != 0) {
    fprintf(stderr, "ngtcp2_crypto_ossl_configure_client_session failed\n");
    return -1;
  }
#endif

  if (ngtcp2_crypto_ossl_ctx_new(&c->ossl_ctx, c->ssl) != 0) {
    fprintf(stderr, "ngtcp2_crypto_ossl_ctx_new failed\n");
    return -1;
  }

  return 0;
}

static int setup_http3(client *c) {
  nghttp3_callbacks h3cb;
  nghttp3_settings h3settings;

  memset(&h3cb, 0, sizeof(h3cb));
  h3cb.stream_close = h3_stream_close_cb;
  h3cb.recv_data = h3_recv_data_cb;
  h3cb.deferred_consume = h3_deferred_consume_cb;
  h3cb.recv_header = h3_recv_header_cb;
  h3cb.stop_sending = h3_stop_sending_cb;
  h3cb.reset_stream = h3_reset_stream_cb;
  h3cb.end_stream = h3_end_stream_cb;

  nghttp3_settings_default(&h3settings);

  int rv = nghttp3_conn_client_new(&c->h3conn, &h3cb, &h3settings, NULL, c);
  if (rv != 0) {
    fprintf(stderr, "nghttp3_conn_client_new: %s\n", nghttp3_strerror(rv));
    return -1;
  }

  return 0;
}

static int setup_quic(client *c) {
  ngtcp2_callbacks cb;
  ngtcp2_settings settings;
  ngtcp2_transport_params params;
  uint8_t dcid_data[18];
  uint8_t scid_data[18];
  ngtcp2_cid dcid;
  ngtcp2_cid scid;

  if (RAND_bytes(dcid_data, (int)sizeof(dcid_data)) != 1 ||
      RAND_bytes(scid_data, (int)sizeof(scid_data)) != 1) {
    fprintf(stderr, "failed to create CID entropy\n");
    return -1;
  }

  ngtcp2_cid_init(&dcid, dcid_data, sizeof(dcid_data));
  ngtcp2_cid_init(&scid, scid_data, sizeof(scid_data));

  memset(&cb, 0, sizeof(cb));
  cb.client_initial = q_client_initial_cb;
  cb.recv_crypto_data = q_recv_crypto_data_cb;
  cb.handshake_completed = q_handshake_completed_cb;
  cb.encrypt = ngtcp2_crypto_encrypt_cb;
  cb.decrypt = ngtcp2_crypto_decrypt_cb;
  cb.hp_mask = ngtcp2_crypto_hp_mask_cb;
  cb.recv_stream_data = q_recv_stream_data_cb;
  cb.acked_stream_data_offset = q_acked_stream_data_offset_cb;
  cb.stream_close = q_stream_close_cb;
  cb.recv_retry = ngtcp2_crypto_recv_retry_cb;
  cb.rand = rand_cb;
  cb.get_new_connection_id = get_new_connection_id_cb;
  cb.update_key = ngtcp2_crypto_update_key_cb;
  cb.delete_crypto_aead_ctx = ngtcp2_crypto_delete_crypto_aead_ctx_cb;
  cb.delete_crypto_cipher_ctx = ngtcp2_crypto_delete_crypto_cipher_ctx_cb;
  cb.get_path_challenge_data = ngtcp2_crypto_get_path_challenge_data_cb;
  cb.version_negotiation = ngtcp2_crypto_version_negotiation_cb;

  ngtcp2_settings_default(&settings);
  settings.initial_ts = timestamp_now(c);

  ngtcp2_transport_params_default(&params);
  params.initial_max_data = 1024 * 1024;
  params.initial_max_stream_data_bidi_local = 512 * 1024;
  params.initial_max_stream_data_bidi_remote = 256 * 1024;
  params.initial_max_stream_data_uni = 256 * 1024;
  params.initial_max_streams_bidi = 16;
  params.initial_max_streams_uni = 16;

  int rv = ngtcp2_conn_client_new(&c->qconn, &dcid, &scid, &c->ps.path,
                                  NGTCP2_PROTO_VER_V1, &cb, &settings,
                                  &params, NULL, c);
  if (rv != 0) {
    fprintf(stderr, "ngtcp2_conn_client_new: %s\n", ngtcp2_strerror(rv));
    return -1;
  }

  ngtcp2_conn_set_tls_native_handle(c->qconn, c->ossl_ctx);

  return 0;
}

static int bind_h3_unidirectional_streams(client *c) {
  int rv;
  int64_t ctrl_id;
  int64_t qenc_id;
  int64_t qdec_id;

  if (c->h3_streams_bound) {
    return 0;
  }

  rv = ngtcp2_conn_open_uni_stream(c->qconn, &ctrl_id, NULL);
  if (rv == NGTCP2_ERR_STREAM_ID_BLOCKED) {
    return 0;
  }
  if (rv != 0) {
    fprintf(stderr, "open control stream: %s\n", ngtcp2_strerror(rv));
    return -1;
  }

  rv = ngtcp2_conn_open_uni_stream(c->qconn, &qenc_id, NULL);
  if (rv != 0) {
    fprintf(stderr, "open qpack encoder stream: %s\n", ngtcp2_strerror(rv));
    return -1;
  }

  rv = ngtcp2_conn_open_uni_stream(c->qconn, &qdec_id, NULL);
  if (rv != 0) {
    fprintf(stderr, "open qpack decoder stream: %s\n", ngtcp2_strerror(rv));
    return -1;
  }

  rv = nghttp3_conn_bind_control_stream(c->h3conn, ctrl_id);
  if (rv != 0) {
    fprintf(stderr, "nghttp3_conn_bind_control_stream: %s\n", nghttp3_strerror(rv));
    return -1;
  }

  rv = nghttp3_conn_bind_qpack_streams(c->h3conn, qenc_id, qdec_id);
  if (rv != 0) {
    fprintf(stderr, "nghttp3_conn_bind_qpack_streams: %s\n", nghttp3_strerror(rv));
    return -1;
  }

  c->h3_streams_bound = 1;
  fprintf(stderr, "bound H3 control stream=%" PRId64 ", qenc=%" PRId64
                  ", qdec=%" PRId64 "\n",
          ctrl_id, qenc_id, qdec_id);
  return 0;
}

static int submit_request(client *c) {
  int rv;
  int64_t stream_id;

  static const uint8_t method[] = "GET";
  static const uint8_t scheme[] = "https";
  static const uint8_t authority[] = REMOTE_HOST;
  static const uint8_t path[] = REQ_PATH;

  nghttp3_nv nva[] = {
    {(uint8_t *)":method", (uint8_t *)method, 7, sizeof(method) - 1,
     NGHTTP3_NV_FLAG_NONE},
    {(uint8_t *)":scheme", (uint8_t *)scheme, 7, sizeof(scheme) - 1,
     NGHTTP3_NV_FLAG_NONE},
    {(uint8_t *)":authority", (uint8_t *)authority, 10,
     sizeof(authority) - 1, NGHTTP3_NV_FLAG_NONE},
    {(uint8_t *)":path", (uint8_t *)path, 5, sizeof(path) - 1,
     NGHTTP3_NV_FLAG_NONE},
    {(uint8_t *)"user-agent", (uint8_t *)"minimal-ngtcp2-nghttp3-client", 10,
     29, NGHTTP3_NV_FLAG_NONE},
  };

  if (c->request_submitted) {
    return 0;
  }

  rv = ngtcp2_conn_open_bidi_stream(c->qconn, &stream_id, NULL);
  if (rv == NGTCP2_ERR_STREAM_ID_BLOCKED) {
    return 0;
  }
  if (rv != 0) {
    fprintf(stderr, "ngtcp2_conn_open_bidi_stream: %s\n", ngtcp2_strerror(rv));
    return -1;
  }

  rv = nghttp3_conn_submit_request(c->h3conn, stream_id, nva,
                                   sizeof(nva) / sizeof(nva[0]), NULL, NULL);
  if (rv != 0) {
    fprintf(stderr, "nghttp3_conn_submit_request: %s\n", nghttp3_strerror(rv));
    return -1;
  }

  c->req_stream_id = stream_id;
  c->request_submitted = 1;
  fprintf(stderr, "submitted GET request on stream=%" PRId64 "\n", stream_id);
  return 0;
}

static int send_quic_packet(client *c, ngtcp2_ssize nwrite, ngtcp2_tstamp ts) {
  if (nwrite <= 0) {
    return 0;
  }
  ssize_t nw = send(c->fd, c->txbuf, (size_t)nwrite, 0);
  if (nw < 0) {
    if (errno == EAGAIN || errno == EWOULDBLOCK) {
      fprintf(stderr, "send would block; aborting this minimal client\n");
      return -1;
    }
    perror("send");
    return -1;
  }
  if ((ngtcp2_ssize)nw != nwrite) {
    fprintf(stderr, "short UDP send\n");
    return -1;
  }
  ++c->tx_pkt_count;
  ngtcp2_conn_update_pkt_tx_time(c->qconn, ts);
  return 0;
}

static int flush_h3_to_quic(client *c) {
  size_t sent_pkts = 0;
  for (;;) {
    if (sent_pkts >= 32) {
      return 0;
    }
    int64_t stream_id = -1;
    int fin = 0;
    nghttp3_vec h3v[8];

    nghttp3_ssize veccnt =
      nghttp3_conn_writev_stream(c->h3conn, &stream_id, &fin, h3v,
                                 sizeof(h3v) / sizeof(h3v[0]));
    if (veccnt < 0) {
      fprintf(stderr, "nghttp3_conn_writev_stream: %s\n",
              nghttp3_strerror((int)veccnt));
      return -1;
    }

    if (veccnt == 0 && stream_id == -1) {
      return 0;
    }

    ngtcp2_vec qv[8];
    for (nghttp3_ssize i = 0; i < veccnt; ++i) {
      qv[i].base = h3v[i].base;
      qv[i].len = h3v[i].len;
    }

    ngtcp2_tstamp ts = timestamp_now(c);
    ngtcp2_ssize pdatalen = -1;
    ngtcp2_ssize nwrite = ngtcp2_conn_writev_stream(
      c->qconn, &c->ps.path, NULL, c->txbuf, sizeof(c->txbuf), &pdatalen,
      fin ? NGTCP2_WRITE_STREAM_FLAG_FIN : NGTCP2_WRITE_STREAM_FLAG_NONE,
      stream_id, veccnt ? qv : NULL, (size_t)veccnt, ts);

    if (nwrite < 0) {
      if (nwrite == NGTCP2_ERR_STREAM_DATA_BLOCKED ||
          nwrite == NGTCP2_ERR_STREAM_SHUT_WR ||
          nwrite == NGTCP2_ERR_STREAM_NOT_FOUND) {
        return 0;
      }
      fprintf(stderr, "ngtcp2_conn_writev_stream(H3): %s\n",
              ngtcp2_strerror((int)nwrite));
      if (nwrite == NGTCP2_ERR_CALLBACK_FAILURE) {
        log_ssl_errors("writev_stream(H3)");
        log_connection_close_reason(c, "writev_stream(H3)");
      }
      return -1;
    }

    if (pdatalen >= 0) {
      int rv = nghttp3_conn_add_write_offset(c->h3conn, stream_id,
                                             (size_t)pdatalen);
      if (rv != 0) {
        fprintf(stderr, "nghttp3_conn_add_write_offset: %s\n",
                nghttp3_strerror(rv));
        return -1;
      }
    }

    if (send_quic_packet(c, nwrite, ts) != 0) {
      return -1;
    }
    ++sent_pkts;

    if (nwrite == 0) {
      return 0;
    }
  }
}

static int flush_quic_control(client *c) {
  size_t sent_pkts = 0;
  for (;;) {
    if (sent_pkts >= 32) {
      return 0;
    }
    ngtcp2_tstamp ts = timestamp_now(c);
    ngtcp2_ssize pdatalen = -1;
    ngtcp2_ssize nwrite = ngtcp2_conn_writev_stream(
      c->qconn, &c->ps.path, NULL, c->txbuf, sizeof(c->txbuf), &pdatalen,
      NGTCP2_WRITE_STREAM_FLAG_NONE, -1, NULL, 0, ts);

    if (nwrite < 0) {
      fprintf(stderr, "ngtcp2_conn_writev_stream(ctrl): %s\n",
              ngtcp2_strerror((int)nwrite));
      if (nwrite == NGTCP2_ERR_CALLBACK_FAILURE) {
        log_ssl_errors("writev_stream(ctrl)");
        log_connection_close_reason(c, "writev_stream(ctrl)");
      }
      return -1;
    }
    if (nwrite == 0) {
      return 0;
    }

    if (send_quic_packet(c, nwrite, ts) != 0) {
      return -1;
    }
    ++sent_pkts;
  }
}

static int drive_tx(client *c) {
  if (flush_h3_to_quic(c) != 0) {
    return -1;
  }

  if (flush_quic_control(c) != 0) {
    return -1;
  }

  return 0;
}

static int read_udp_once(client *c) {
  ssize_t nread = recv(c->fd, c->rxbuf, sizeof(c->rxbuf), 0);
  if (nread < 0) {
    if (errno == EAGAIN || errno == EWOULDBLOCK) {
      return 0;
    }
    perror("recv");
    return -1;
  }

  ngtcp2_tstamp ts = timestamp_now(c);
  ++c->rx_pkt_count;
  int rv = ngtcp2_conn_read_pkt(c->qconn, &c->ps.path, NULL,
                                c->rxbuf, (size_t)nread, ts);
  if (rv != 0) {
    if (rv == NGTCP2_ERR_DRAINING || rv == NGTCP2_ERR_CLOSING) {
      log_connection_close_reason(c, "ngtcp2_conn_read_pkt");
    }
    fprintf(stderr, "ngtcp2_conn_read_pkt: %s\n", ngtcp2_strerror(rv));
    return -1;
  }

  return 1;
}

static int run(client *c) {
  struct pollfd pfd;
  ngtcp2_tstamp started_ts = timestamp_now(c);

  pfd.fd = c->fd;
  pfd.events = POLLIN;

  /* Kick off Initial packets / TLS ClientHello. */
  if (drive_tx(c) != 0) {
    return -1;
  }

  while (!c->had_error && !c->response_complete) {
    ngtcp2_tstamp now = timestamp_now(c);
    ngtcp2_tstamp expiry = ngtcp2_conn_get_expiry(c->qconn);
    int timeout_ms = 50;

    if (expiry <= now) {
      timeout_ms = 0;
    } else {
      uint64_t diff_ns = expiry - now;
      timeout_ms = (int)(diff_ns / NGTCP2_MILLISECONDS);
      if (timeout_ms > 50) {
        timeout_ms = 50;
      }
    }

    int n = poll(&pfd, 1, timeout_ms);
    if (n < 0) {
      if (errno == EINTR) {
        continue;
      }
      perror("poll");
      return -1;
    }

    if (n > 0 && (pfd.revents & POLLIN)) {
      for (;;) {
        int rr = read_udp_once(c);
        if (rr < 0) {
          return -1;
        }
        if (rr == 0) {
          break;
        }
      }
    }

    now = timestamp_now(c);
    if (ngtcp2_conn_handle_expiry(c->qconn, now) != 0) {
      fprintf(stderr, "ngtcp2_conn_handle_expiry failed\n");
      return -1;
    }

    if (ngtcp2_conn_in_closing_period(c->qconn) ||
        ngtcp2_conn_in_draining_period(c->qconn)) {
      fprintf(stderr, "QUIC connection is closing/draining\n");
      return -1;
    }

    if (c->handshake_completed) {
      if (bind_h3_unidirectional_streams(c) != 0) {
        return -1;
      }
      if (c->h3_streams_bound && submit_request(c) != 0) {
        return -1;
      }
    }

    if (drive_tx(c) != 0) {
      return -1;
    }

    if (timestamp_now(c) - started_ts > 30 * NGTCP2_SECONDS) {
      fprintf(stderr,
              "timeout: no complete response in 30s "
              "(handshake=%d, bound=%d, submitted=%d, tx_pkts=%" PRIu64
              ", rx_pkts=%" PRIu64 ")\n",
              c->handshake_completed, c->h3_streams_bound, c->request_submitted,
              c->tx_pkt_count, c->rx_pkt_count);
      return -1;
    }
  }

  return c->had_error ? -1 : 0;
}

static void cleanup(client *c) {
  if (c->h3conn) {
    nghttp3_conn_del(c->h3conn);
  }
  if (c->qconn) {
    ngtcp2_conn_del(c->qconn);
  }
  if (c->ossl_ctx) {
    ngtcp2_crypto_ossl_ctx_del(c->ossl_ctx);
  }
  if (c->ssl) {
    SSL_set_app_data(c->ssl, NULL);
    SSL_free(c->ssl);
  }
  if (c->ssl_ctx) {
    SSL_CTX_free(c->ssl_ctx);
  }
  if (c->fd >= 0) {
    close(c->fd);
  }
}

int main(void) {
  client c;
  memset(&c, 0, sizeof(c));
  c.fd = -1;
  c.req_stream_id = -1;

  if (socket_connect(&c) != 0) {
    cleanup(&c);
    return 1;
  }

  if (setup_tls(&c) != 0) {
    cleanup(&c);
    return 1;
  }

  if (setup_http3(&c) != 0) {
    cleanup(&c);
    return 1;
  }

  if (setup_quic(&c) != 0) {
    cleanup(&c);
    return 1;
  }

  int rv = run(&c);
  cleanup(&c);

  if (rv != 0) {
    return 1;
  }
  return 0;
}