epoll illumos bug

main.c

#include <err.h>
#include <pthread.h>
#include <stdbool.h>
#include <stdio.h>
#include <unistd.h>
#include <sys/epoll.h>
#include <sys/eventfd.h>

void write_to_event_fd(int event_fd) {
	static uint64_t data = 1;

	if (write(event_fd, &data, sizeof (data)) != sizeof (data)) {
		err(1, "write to event_fd");
	}
}

void read_from_event_fd(int event_fd) {
	static uint64_t want_data = 1;
	uint64_t have_data;

	if (read(event_fd, &have_data, sizeof (have_data)) != sizeof (have_data)) {
		err(1, "read from event_fd");
	}

	if (want_data != have_data) {
		err(1, "read bad data from event_fd");
	}
}

void wait_for_epoll_events(int epoll_fd) {
	int nevents = 50;
	int timeout = -1;
	struct epoll_event ep_events[nevents];
	int num_events;

	num_events = epoll_wait(epoll_fd, ep_events, nevents, timeout);

	if (num_events != 1) {
		err(1, "epoll_wait returned bad value: %d", num_events);
	}
}

int main(int arg, char **argv) {
	int ret;
	struct epoll_event ev;

	// create epoll handle
	int epoll_fd = epoll_create1(EPOLL_CLOEXEC);
	if (epoll_fd == -1) {
		err(1, "epoll_create1");
	}

	// create eventfd handle
	int event_fd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
	if (event_fd == -1) {
		err(1, "eventfd create");
	}

	// add eventfd handle to epoll event watcher
	ev.events = EPOLLIN | EPOLLRDHUP | EPOLLET;
	ev.data.ptr = NULL;
	ret = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, event_fd, &ev);
	if (ret == -1) {
		err(1, "epoll_ctl (add eventfd to epoll)");
	}

	write_to_event_fd(event_fd);
	wait_for_epoll_events(epoll_fd);
	printf("first epoll_wait returned\n");
	/*
	 * If we skip the read on Linux the edge triggered event
	 * still causes epoll_wait to return.  On illumos this isn't
	 * the case
	 */
	//read_from_event_fd(event_fd);
	write_to_event_fd(event_fd);
	wait_for_epoll_events(epoll_fd);
	printf("second epoll_wait returned\n");

	return 0;
}

eventfd has been swapped out for pipe and this reveals most likely an epoll edge case -> https://gist.github.com/bahamas10/4d88c3578f46ebad807cdf87c4419b1c

#include <err.h>
#include <fcntl.h>
#include <pthread.h>
#include <stdbool.h>
#include <stdio.h>
#include <unistd.h>
#include <sys/epoll.h>
#include <sys/eventfd.h>

void write_to_fd(int fd) {
	static uint64_t data = 1;

	if (write(fd, &data, sizeof (data)) != sizeof (data)) {
		err(1, "write to fd");
	}
}

void read_from_fd(int fd) {
	static uint64_t want_data = 1;
	uint64_t have_data;

	if (read(fd, &have_data, sizeof (have_data)) != sizeof (have_data)) {
		err(1, "read from fd");
	}

	if (want_data != have_data) {
		err(1, "read bad data from fd");
	}
}

void wait_for_epoll_events(int epoll_fd) {
	int nevents = 50;
	int timeout = -1;
	struct epoll_event ep_events[nevents];
	int num_events;

	num_events = epoll_wait(epoll_fd, ep_events, nevents, timeout);

	if (num_events != 1) {
		err(1, "epoll_wait returned bad value: %d", num_events);
	}
}

int main(int arg, char **argv) {
	int ret;
	struct epoll_event ev;

	// create epoll handle
	int epoll_fd = epoll_create1(EPOLL_CLOEXEC);
	if (epoll_fd == -1) {
		err(1, "epoll_create1");
	}

	int pipe_fds[2];
	if (pipe(pipe_fds) == -1) {
		err(1, "pipe");
	}
	if (fcntl(pipe_fds[0], F_SETFL, O_NONBLOCK) == -1) {
		err(3, "set read end nonblocking");
	}
	if (fcntl(pipe_fds[1], F_SETFL, O_NONBLOCK) == -1) {
		err(3, "set write end nonblocking");
	}

	// add eventfd handle to epoll event watcher
	ev.events = EPOLLIN | EPOLLRDHUP | EPOLLET;
	ev.data.ptr = NULL;
	ret = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, pipe_fds[0], &ev);
	if (ret == -1) {
		err(1, "epoll_ctl (add pipe to epoll)");
	}

	printf("writing to pipe...\n");
	write_to_fd(pipe_fds[1]);
	printf("waiting for epoll...\n");
	wait_for_epoll_events(epoll_fd);
	printf("first epoll_wait returned\n");
	/*
	 * If we skip the read on Linux the edge triggered event
	 * still causes epoll_wait to return.  On illumos this isn't
	 * the case
	 */
	//read_from_fd(pipe_fds[0]);
	printf("writing to pipe...\n");
	write_to_fd(pipe_fds[1]);
	printf("waiting for epoll...\n");
	wait_for_epoll_events(epoll_fd);
	printf("second epoll_wait returned\n");

	return 0;
}