TCP test (sometimes connect() fails on SmartOS, but not on OmniOS or OSX)

gistfile1.c

/*
 * Verify fundamental behavior of TCP stack.
 *
 * Based on a potential bug found when running libMicro's close_tcp
 * and pipe benchmarks on SmartOS.
 *
 * https://github.com/rzezeski/libMicro/blob/master/src/close_tcp.c
 *
 * https://github.com/rzezeski/libMicro/blob/master/src/pipe.c
 *
 */

/* #include <netinet/in.h> */
#include <sys/socket.h>
#include <sys/types.h>
#include <arpa/inet.h>
#include <netinet/tcp.h>
#include <errno.h>
#include <ifaddrs.h>
#include <netdb.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#define TT_BUF_LEN	16
#define	TT_PORT		12345
#define	TT_CHK(exp)							\
	if (exp) {} else tt_err(errno, __FILE__, __LINE__)

void
tt_err(int errnum, char *file, int line)
{
	/* Log where and why test failed. */
	printf("ERROR %d %s %s:%d\n",
	    errnum, strerror(errnum), file, line);
	fflush(stdout);
	exit(1);
}

int
main(int argc, char **argv)
{
	int			lsn;	/* listen socket */
	int			conn;	/* active side of socket */
	int			acc;	/* passive side of socket */
	int			opt;	/* setsockopt() option val */
	struct sockaddr_in	addr;	/* listen/connect address */
	struct sockaddr_in	acc_addr; /* accept address */
	struct hostent		*host;	/* hosts entry */
	socklen_t		size;	/* needed for accept() */

	int			ret;	/* return value */
	int			iters;	/* num iterations */
	int			pkts;	/* num packets per iter */

	char			wbuf[TT_BUF_LEN]; /* write buffer */
	char			rbuf[TT_BUF_LEN]; /* read buffer */
	char			*ifname;	  /* interface to bind to */
	int			uopt;		  /* user option */

	iters = 2000;
	pkts = 100;
	ifname = NULL;

	while ((uopt = getopt(argc, argv, "i:p:I:")) != -1) {
		switch (uopt) {
		case 'i':
			iters = atoi(optarg);
			break;
		case 'p':
			pkts = atoi(optarg);
			break;
		case 'I':
			ifname = optarg;
			break;
		}
	}

	/*
	 * Step 1: Setup listener socket. This is only done once.
	 */

	TT_CHK((lsn = socket(AF_INET, SOCK_STREAM, 0)) != -1);
	opt = 1;
	TT_CHK(setsockopt(lsn, SOL_SOCKET, SO_REUSEADDR,
		&opt, sizeof (int)) != -1);

	struct ifaddrs *ifaddrs, *ifa;
	TT_CHK(getifaddrs(&ifaddrs) == 0);
	memset(&addr, 0, sizeof (struct sockaddr_in));

	if (ifname != NULL) {
		for (ifa = ifaddrs; ifa != NULL; ifa = ifa->ifa_next) {
			if ((memcmp(ifname, ifa->ifa_name,
				    strlen(ifname)) == 0) &&
			    (ifa->ifa_addr->sa_family == AF_INET)) {

				printf("select interface %s\n", ifa->ifa_name);

				addr.sin_family = AF_INET;
				addr.sin_port = htons(TT_PORT);
				struct sockaddr_in *sa;
				sa = (struct sockaddr_in *)ifa->ifa_addr;
				addr.sin_addr = sa->sin_addr;

				break;
			}
		}
	} else {
		TT_CHK((host = gethostbyname("localhost")) != NULL);

		addr.sin_family = AF_INET;
		addr.sin_port = htons(TT_PORT);
		memcpy(&addr.sin_addr.s_addr, host->h_addr_list[0],
		    sizeof (struct in_addr));
	}

	/* If this trips failed to select interface. */
	TT_CHK(addr.sin_port != 0);

	ret = bind(lsn, (struct sockaddr *)&addr, sizeof (struct sockaddr_in));
	TT_CHK(ret == 0);

	/* Listen on socket with backlog of 1. */
	TT_CHK(listen(lsn, 1) == 0);

	/*
	 * Step 2: Run test loop.
	 */

	for (int i = 0; i < iters; i++) {
		printf("Iteration %d...", i + 1);

		/*
		 * Step 2.a: Initiate connect request (active side),
		 *	     since there is a backlog of 1 this SHOULD
		 *	     always succeed.
		 */
		ret = 0;

		TT_CHK((conn = socket(AF_INET, SOCK_STREAM, 0)) != -1);
		/* Disable nagle (don't coalesce small packets). */
		TT_CHK(setsockopt(conn, IPPROTO_TCP,
			TCP_NODELAY, &opt, sizeof (int)) == 0);

		ret = connect(conn, (struct sockaddr *)&addr,
		    sizeof (struct sockaddr_in));
		TT_CHK(ret == 0);

		/*
		 * Step 2.b: Accept the pending connect request (passive side).
		 */
		size = sizeof (struct sockaddr);
		acc = accept(lsn, (struct sockaddr *)&acc_addr, &size);
		TT_CHK(acc != -1);

		/*
		 * Step 2.c: Send N packets acorss the wire, each
		 *	     packet containing the string
		 *	     representation of the iteration number.
		 */
		for (int i = 0; i < pkts; i++) {
			TT_CHK(snprintf(wbuf, TT_BUF_LEN, "%d", i) > 0);
			int size = strlen(wbuf);
			TT_CHK(write(conn, wbuf, strlen(wbuf)) == size);

		}

		/*
		 * Step 2.d: Verify that N packets went across the
		 *	     socket and in the correct order.
		 */
		for (int i = 0; i < pkts; i++) {
			TT_CHK(snprintf(wbuf, TT_BUF_LEN, "%d", i) > 0);
			int size = strlen(wbuf);
			TT_CHK(read(acc, rbuf, size) == size);
			TT_CHK(memcmp(rbuf, wbuf, size) == 0);

		}

		/*
		 * Step 2.e: Close the passive side.
		 */
		TT_CHK(close(acc) == 0);

		/*
		 * Step 2.f: Close the active side.
		 */
		TT_CHK(close(conn) == 0);

		printf("OK\n");
	}

	/*
	 * Step 3: Close listener and return success.
	 */
	TT_CHK(close(lsn) == 0);

	return (0);
}

By default this will run 2000 iterations each sending 100 packets over the loopback interface.

I compiled on SmartOS with:

gcc -std=c99 -m64 -Wall -Wextra -Wno-unused-parameter -Werror -O2 -D_REENTRANT -D__EXTENSIONS__ -o tcp-test tcp-test.c -lsocket -lnsl

Compiled on OSX with:

cc -std=c99 -m64 -Wall -Wextra -Wno-unused-parameter -Werror -O2 -D_REENTRANT -o tcp-test tcp-test.c

Specify a different interface use -I.

Specify number of iterations with -i.

Specify number of packets with -p.

./tcp-test -i 2000 -p 100 -I net0