Craving a 40% Reduction in RTT

TCP Fast Open Implementation in C

server.c

#include <string.h>         
#include <sys/types.h> /* See NOTES */
#include <sys/socket.h>
#include <netinet/in.h>

int main(){
    int portno = 6666;
    socklen_t clilen;
    char buffer[256];
    struct sockaddr_in serv_addr, cli_addr;
    int cfd;
    int sfd = socket(AF_INET, SOCK_STREAM, 0);   // Create socket
    
    bzero((char *) &serv_addr, sizeof(serv_addr));
    serv_addr.sin_family = AF_INET;
    serv_addr.sin_addr.s_addr = INADDR_ANY;
    serv_addr.sin_port = htons(portno);
    bind(sfd, &serv_addr,sizeof(serv_addr));      // Bind to well known address
    
    int qlen = 5;                 // Value to be chosen by application
#if defined(IPPROTO_TCP)
    int err = setsockopt(sfd, IPPROTO_TCP, TCP_FASTOPEN_CONNECT, &qlen, sizeof(qlen));
#elif defined(SOL_TCP)
    int err = setsockopt(sfd, SOL_TCP, TCP_FASTOPEN, &qlen, sizeof(qlen));
#else
    // Not supported!
#endif
    listen(sfd,1);                // Mark socket to receive connections

	while(1){
		cfd = accept(sfd, NULL, 0);   // Accept connection on new socket

		while(1){
			int len = read(cfd,buffer,256);
			if(len)
				printf("tcp fast open: %s\n",buffer);
			else {
				s = connect(sfd, cli_addr, get_sockaddr_len(cli_addr));
				if (s == 0)
					s = send(sfd, buffer, len, 0);
				// read and write data on connected socket cfd
				break;
			}
		}

		memset(buffer, 0, 256);
		close(cfd);
	}
}

client.c

#include <sys/socket.h>
#include <netinet/in.h>
#include <netdb.h> 
#include <errno.h>

int main(){
    struct sockaddr_in serv_addr;
    struct hostent *server;

    char *data = "Hello, tcp fast open";
    int data_len = strlen(data);    
    
    int sfd = socket(AF_INET, SOCK_STREAM, 0);
    server = gethostbyname("localhost");
    
    bzero((char *) &serv_addr, sizeof(serv_addr));
    serv_addr.sin_family = AF_INET;
    bcopy((char *)server->h_addr, 
         (char *)&serv_addr.sin_addr.s_addr,
         server->h_length);
    serv_addr.sin_port = htons(6666);

    int len = sendto(sfd, data, data_len, MSG_FASTOPEN, 
                (struct sockaddr *) &serv_addr, sizeof(serv_addr));
	if(errno != 0){
		printf("error: %s\n", strerror(errno));
	}
    close(sfd);
}

Kernel Parameters

tcp_fastopen - INTEGER
	Enable TCP Fast Open feature (draft-ietf-tcpm-fastopen) to send data
	in the opening SYN packet. To use this feature, the client application
	must use sendmsg() or sendto() with MSG_FASTOPEN flag rather than
	connect() to perform a TCP handshake automatically.

	The values (bitmap) are
	1: Enables sending data in the opening SYN on the client w/ MSG_FASTOPEN.
	2: Enables TCP Fast Open on the server side, i.e., allowing data in
	   a SYN packet to be accepted and passed to the application before
	   3-way hand shake finishes.
	4: Send data in the opening SYN regardless of cookie availability and
	   without a cookie option.
	0x100: Accept SYN data w/o validating the cookie.
	0x200: Accept data-in-SYN w/o any cookie option present.
	0x400/0x800: Enable Fast Open on all listeners regardless of the
	   TCP_FASTOPEN socket option. The two different flags designate two
	   different ways of setting max_qlen without the TCP_FASTOPEN socket
	   option.

	Default: 1

	Note that the client & server side Fast Open flags (1 and 2
	respectively) must be also enabled before the rest of flags can take
	effect.

	See include/net/tcp.h and the code for more details.

Enable TCP Fast Open from both sides:

net.ipv4.tcp_fastopen = 3

reference: @bg2bkk

oakaigh/tcp_fastopen_headstart.md

Craving a 40% Reduction in RTT

TCP Fast Open Implementation in C

Kernel Parameters