Linux kTLS TLS 1.3 TX device-offload sendfile() final-record flush reproducer

ktls_sendfile_final_record_repro.c

// SPDX-License-Identifier: GPL-2.0
/*
 * ktls_sendfile_selftest.c
 *
 * Self-contained reproducer for a TLS 1.3 kTLS HW/device-offload sendfile()
 * final-record flush/loss issue, in the style of
 * tools/testing/selftests/net/tls: kTLS is installed with raw
 * setsockopt(SOL_TLS, TLS_TX/TLS_RX) using fixed test crypto material -- no TLS
 * handshake, no OpenSSL, no Go. This matches how the kernel selftest and
 * userspace stacks (e.g. a server that handshakes in userspace and then hands
 * the socket to the kernel) install kTLS, and isolates the kernel/driver from
 * any library.
 *
 * One binary, two roles:
 *   server (-l):  accept a TCP connection, TCP_ULP=tls, setsockopt(TLS_TX),
 *                 then send a file with the raw sendfile(2) syscall and close.
 *   client (-c):  connect, TCP_ULP=tls, setsockopt(TLS_RX) with the SAME keys,
 *                 read the decrypted stream, report bytes received and any
 *                 decrypt/read error.
 *
 * IMPORTANT: TLS_HW (NIC inline crypto) only engages over a real
 * offload-capable interface. A loopback / same-host connection runs TLS_SW and
 * will NOT show the bug. Run the server on the offload node and the client on a
 * separate host on the same L2/L3 path (so TX egresses the ConnectX with
 * tls-hw-tx-offload on).
 *
 * Body send modes:
 *   over  (default): sendfile() with count > remaining file bytes (~2 GiB, what
 *                    a copy loop / Go's ReadFrom passes for an unknown size).
 *   exact (-e):      sendfile() with count == remaining bytes.
 * Teardown:
 *   -n : send a close_notify alert (raw sendmsg, TLS_SET_RECORD_TYPE) before
 *        close; default is an abrupt close().
 * -z : also set TLS_TX_ZEROCOPY_RO (zero-copy sendfile, the device splice
 * path).
 *
 * Observed (ConnectX-6 Dx, v14 series; file 226965 B = 13*16384 + 13973;
 * confirm TlsTxDevice climbs in /proc/net/tls_stat on the server for HW rows):
 *
 *   TLS_HW  count>EOF  close()                      -> client gets 212992 (last record lost)
 *   TLS_HW  count>EOF  close()  without -z          -> client gets 212992 (zerocopy irrelevant)
 *   TLS_HW  exact      close()  (-e)                -> client gets 226965 (complete)
 *   TLS_HW  count>EOF  close_notify  (-n)           -> client gets 226965 (complete)
 *   TLS_SW  count>EOF  close()  (hw-tx-offload off) -> client gets 226965 (complete)
 *
 * So: TLS_HW does not finalize/flush the connection's final record of a
 * count > EOF sendfile at EOF; an abrupt close discards it. A trailing write
 * (close_notify), an exact count, or software kTLS all avoid it.
 *
 * Build:  cc -O2 -o ktls_sendfile_selftest ktls_sendfile_selftest.c
 * File:   needs a partial final record (size % 16384 != 0), e.g.
 *           dd if=/dev/urandom of=/tmp/testfile bs=226965 count=1
 * Server: ./ktls_sendfile_selftest -l -f /tmp/testfile [-e] [-n] [-z]
 * Client: ./ktls_sendfile_selftest -c -H <server-ipv6>
 *         default port 8443; -p <port> overrides and MUST match on both ends.
 */

#define _GNU_SOURCE
#include <arpa/inet.h>
#include <errno.h>
#include <fcntl.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <poll.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/sendfile.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <unistd.h>

#include <linux/tls.h>

/* Fallbacks so this builds without a recent uapi <linux/tls.h>. */
#ifndef SOL_TLS
#define SOL_TLS 282
#endif
#ifndef TCP_ULP
#define TCP_ULP 31
#endif
#ifndef TLS_TX
#define TLS_TX 1
#endif
#ifndef TLS_RX
#define TLS_RX 2
#endif
#ifndef TLS_TX_ZEROCOPY_RO
#define TLS_TX_ZEROCOPY_RO 3
#endif
#ifndef TLS_SET_RECORD_TYPE
#define TLS_SET_RECORD_TYPE 1
#endif
#ifndef TLS_GET_RECORD_TYPE
#define TLS_GET_RECORD_TYPE 2
#endif
#ifndef TLS_1_3_VERSION
#define TLS_1_3_VERSION 0x0304
#endif
#ifndef TLS_CIPHER_AES_GCM_128
#define TLS_CIPHER_AES_GCM_128 51
#endif

#define ALERT_RECORD 21

/* Fixed TLS 1.3 AES-128-GCM test crypto material -- identical on both ends,
 * rec_seq starts at 0 (kTLS is installed on a fresh connection, so the first
 * application record is sequence 0). Test vectors, not secret. */
static const unsigned char KEY[TLS_CIPHER_AES_GCM_128_KEY_SIZE] = {
    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
    0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f};
static const unsigned char IV[TLS_CIPHER_AES_GCM_128_IV_SIZE] = {
    0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17};
static const unsigned char SALT[TLS_CIPHER_AES_GCM_128_SALT_SIZE] = {
    0x20, 0x21, 0x22, 0x23};

static void fill_crypto_info(struct tls12_crypto_info_aes_gcm_128 *ci) {
  memset(ci, 0, sizeof *ci);
  ci->info.version = TLS_1_3_VERSION;
  ci->info.cipher_type = TLS_CIPHER_AES_GCM_128;
  memcpy(ci->key, KEY, sizeof KEY);
  memcpy(ci->iv, IV, sizeof IV);
  memcpy(ci->salt, SALT, sizeof SALT);
  /* ci->rec_seq stays all-zero */
}

/* Install kTLS for one direction (TLS_TX or TLS_RX) with the shared keys. */
static int enable_ktls(int fd, int dir) {
  if (setsockopt(fd, IPPROTO_TCP, TCP_ULP, "tls", sizeof "tls") < 0) {
    perror("setsockopt TCP_ULP tls");
    return -1;
  }
  struct tls12_crypto_info_aes_gcm_128 ci;
  fill_crypto_info(&ci);
  if (setsockopt(fd, SOL_TLS, dir, &ci, sizeof ci) < 0) {
    perror(dir == TLS_TX ? "setsockopt TLS_TX" : "setsockopt TLS_RX");
    return -1;
  }
  return 0;
}

/* close_notify the way userspace stacks do over kTLS: a raw sendmsg with
 * TLS_SET_RECORD_TYPE=alert, payload {warning(1), close_notify(0)}. */
static void send_close_notify(int fd) {
  unsigned char alert[2] = {1, 0};
  char cbuf[CMSG_SPACE(1)];
  struct iovec iov = {alert, sizeof alert};
  struct msghdr msg;
  memset(&msg, 0, sizeof msg);
  memset(cbuf, 0, sizeof cbuf);
  msg.msg_iov = &iov;
  msg.msg_iovlen = 1;
  msg.msg_control = cbuf;
  msg.msg_controllen = sizeof cbuf;
  struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
  cmsg->cmsg_level = SOL_TLS;
  cmsg->cmsg_type = TLS_SET_RECORD_TYPE;
  cmsg->cmsg_len = CMSG_LEN(1);
  *CMSG_DATA(cmsg) = ALERT_RECORD;
  if (sendmsg(fd, &msg, 0) < 0)
    perror("close_notify sendmsg");
}

static int run_server(int port, const char *file, int over, int closenotify,
                      int zerocopy) {
  int ls = socket(AF_INET6, SOCK_STREAM, 0);
  if (ls < 0) {
    perror("socket");
    return 1;
  }
  int one = 1;
  setsockopt(ls, SOL_SOCKET, SO_REUSEADDR, &one, sizeof one);
  struct sockaddr_in6 a;
  memset(&a, 0, sizeof a);
  a.sin6_family = AF_INET6;
  a.sin6_addr = in6addr_any;
  a.sin6_port = htons(port);
  if (bind(ls, (struct sockaddr *)&a, sizeof a) < 0) {
    perror("bind");
    return 1;
  }
  if (listen(ls, 16) < 0) {
    perror("listen");
    return 1;
  }
  fprintf(stderr, "server :%d mode=%s closenotify=%d zerocopy=%d file=%s\n",
          port, over ? "over" : "exact", closenotify, zerocopy, file);

  for (;;) {
    int c = accept(ls, NULL, NULL);
    if (c < 0)
      continue;

    if (enable_ktls(c, TLS_TX) < 0) {
      close(c);
      continue;
    }
    if (zerocopy) {
      int v = 1;
      if (setsockopt(c, SOL_TLS, TLS_TX_ZEROCOPY_RO, &v, sizeof v) < 0)
        perror("setsockopt TLS_TX_ZEROCOPY_RO");
    }

    int fd = open(file, O_RDONLY);
    if (fd < 0) {
      perror("open");
      close(c);
      continue;
    }
    struct stat st;
    if (fstat(fd, &st) < 0) {
      perror("fstat");
      close(fd);
      close(c);
      continue;
    }

    /* Non-blocking + poll, so the transfer is split across several sendfile
     * calls (netpoller-style), like Go's net.TCPConn.ReadFrom. offset = NULL:
     * the kernel uses/advances the file position. */
    int fl = fcntl(c, F_GETFL, 0);
    if (fl < 0 || fcntl(c, F_SETFL, fl | O_NONBLOCK) < 0) {
      perror("fcntl O_NONBLOCK");
      close(fd);
      close(c);
      continue;
    }
    long long total = 0;
    const long long over_count = 1LL << 31; /* ~2 GiB, like maxSendfileSize */
    for (;;) {
      size_t count = over ? (size_t)over_count : (size_t)(st.st_size - total);
      ssize_t n = sendfile(c, fd, NULL, count);
      if (n > 0) {
        total += n;
        if (!over && total >= st.st_size)
          break;
        continue;
      }
      if (n == 0)
        break; /* EOF (over-mode loops here after the file is exhausted) */
      if (errno == EAGAIN || errno == EWOULDBLOCK) {
        struct pollfd p = {c, POLLOUT, 0};
        poll(&p, 1, 1000);
        continue;
      }
      perror("sendfile");
      break;
    }
    (void)fcntl(c, F_SETFL, fl); /* restore blocking; best effort */
    fprintf(stderr, "served file_size=%lld sent=%lld\n", (long long)st.st_size,
            total);

    if (closenotify)
      send_close_notify(c);
    close(fd);
    close(c);
  }
}

/* Receive one TLS record's worth of plaintext via recvmsg, reporting the record
 * content type from the TLS_GET_RECORD_TYPE cmsg (kTLS RX delivers control
 * records, e.g. close_notify, this way; a plain recv() would fail them with
 * EIO). Returns the byte count (0 = TCP FIN, -1 = error) and the record type in
 * *rectype (defaults to application_data when the kernel attaches no cmsg). */
static ssize_t recv_record(int fd, char *buf, size_t len, unsigned char *rectype) {
  char oob[CMSG_SPACE(1)];
  struct iovec iov = {buf, len};
  struct msghdr msg;
  memset(&msg, 0, sizeof msg);
  msg.msg_iov = &iov;
  msg.msg_iovlen = 1;
  msg.msg_control = oob;
  msg.msg_controllen = sizeof oob;
  *rectype = 23; /* application_data unless a cmsg says otherwise */
  ssize_t n = recvmsg(fd, &msg, 0);
  if (n <= 0)
    return n;
  for (struct cmsghdr *c = CMSG_FIRSTHDR(&msg); c; c = CMSG_NXTHDR(&msg, c)) {
    if (c->cmsg_level == SOL_TLS && c->cmsg_type == TLS_GET_RECORD_TYPE)
      *rectype = *CMSG_DATA(c);
  }
  return n;
}

static int run_client(const char *host, int port) {
  int s = socket(AF_INET6, SOCK_STREAM, 0);
  if (s < 0) {
    perror("socket");
    return 1;
  }
  struct sockaddr_in6 a;
  memset(&a, 0, sizeof a);
  a.sin6_family = AF_INET6;
  a.sin6_port = htons(port);
  if (inet_pton(AF_INET6, host, &a.sin6_addr) != 1) {
    fprintf(stderr, "client: -H must be an IPv6 literal (got %s)\n", host);
    return 2;
  }
  if (connect(s, (struct sockaddr *)&a, sizeof a) < 0) {
    perror("connect");
    return 1;
  }
  if (enable_ktls(s, TLS_RX) < 0) {
    close(s);
    return 1;
  }

  long long total = 0;
  char buf[65536];
  int err = 0;
  const char *how = "TCP FIN (no close_notify)";
  for (;;) {
    unsigned char rt;
    ssize_t n = recv_record(s, buf, sizeof buf, &rt);
    if (n < 0) {
      /* Depending on the failure mode, the peer may either receive a short
       * stream or fail while decrypting/processing the final record. */
      fprintf(stderr, "recv error after %lld bytes: %s (errno=%d)\n", total,
              strerror(errno), errno);
      err = errno;
      how = "recv error";
      break;
    }
    if (n == 0) {
      break; /* TCP FIN without a TLS close_notify */
    }
    if (rt == 23) { /* application_data */
      total += n;
      continue;
    }
    if (rt == ALERT_RECORD) { /* level, desc */
      how = (n >= 2 && buf[1] == 0) ? "close_notify (clean TLS shutdown)"
                                    : "TLS alert";
      break;
    }
    fprintf(stderr, "unexpected TLS record type %d\n", rt);
    how = "unexpected record";
    break;
  }
  printf("client received %lld bytes, end=%s (recv_errno=%d)\n", total, how, err);
  close(s);
  return 0;
}

int main(int argc, char **argv) {
  signal(SIGPIPE, SIG_IGN);

  int listen_mode = 0, client_mode = 0;
  int port = 8443, over = 1, closenotify = 0, zerocopy = 0;
  const char *file = NULL, *host = NULL;

  int opt;
  while ((opt = getopt(argc, argv, "lcH:p:f:enz")) != -1) {
    switch (opt) {
    case 'l':
      listen_mode = 1;
      break;
    case 'c':
      client_mode = 1;
      break;
    case 'H':
      host = optarg;
      break;
    case 'p':
      port = atoi(optarg);
      break;
    case 'f':
      file = optarg;
      break;
    case 'e':
      over = 0;
      break; /* exact count instead of over */
    case 'n':
      closenotify = 1;
      break;
    case 'z':
      zerocopy = 1;
      break;
    default:
      fprintf(
          stderr,
          "usage:\n"
          "  server: %s -l -p <port> -f <file> [-e] [-n] [-z]\n"
          "  client: %s -c -H <server-ipv6> -p <port>\n"
          "  -e exact count   -n send close_notify   -z TLS_TX_ZEROCOPY_RO\n",
          argv[0], argv[0]);
      return 2;
    }
  }

  if (listen_mode && file)
    return run_server(port, file, over, closenotify, zerocopy);
  if (client_mode && host)
    return run_client(host, port);

  fprintf(stderr,
          "specify either -l -f <file> (server) or -c -H <ip> (client)\n");
  return 2;
}