bahamas10 · March 21, 2022 20:26 · bahamas10 · Mar 21, 2022
diff --git a/00-readme.md b/00-readme.md
diff --git a/eventfd-test.c b/eventfd-test.c
 /*
 * Simple test program to exercise possible LX bug wrt epoll/event2fd.
 *
 * This program works be creating an epoll fd and eventfd fd.  The eventfd fd is
 * registered to be watched by the epoll fd, and then 2 threads are spun up to
 * communicate with eachother via the eventfd.
 *
 * Thread1:
 *   - writes to the event fd
 *   - waits for a condition signal to continue
 *   - loop
 * Thread2:
 *   - waits for epoll events on the event fd
 *   - (optional) reads the data from the event fd
 *   - signals thread 1 to wakeup
 *   - loops
 *
 * The optional step of reading the data in Thread2 is what can trigger the bug.
 * If the data is read, this program works without issue.  If the data is not
 * read, epoll will only generate an event for the first write to the eventfd,
 * but not subsequent writes.
 *
 * There are 2 #define's that can modify this programs behavior
 *   - NUM_ITERATIONS - number of times to loop for each thread, should be >=2
 *                      to exercise this bug.
 *   - READ_EVENT_FD - whether to read the data in Thread2 (the optional step),
 *                     setting this to `false` will exercise the bug and at this
 *                     current moment result in a hang after the 2nd call to
 *                     write.
 *
 * More info on bug origin:
 * https://gist.github.com/bahamas10/497acef27b8519e824e90d9eb648a6f6
 *
 * Author: Dave Eddy <[email protected]>
 * Date: March 19, 2022
 * License: MIT
 */

 #include <err.h>
 #include <pthread.h>
 #include <stdbool.h>
 #include <stdio.h>
 #include <unistd.h>
 #include <sys/epoll.h>
 #include <sys/eventfd.h>

 #define READ_EVENT_FD false
 #define NUM_ITERATIONS 5

 // shared with threads
 struct context {
        int epoll_fd;
        int event_fd;
        pthread_mutex_t *lock;
        pthread_cond_t *cond;
 };

 void write_to_event_fd(int event_fd) {
        static uint64_t data = 1;

        if (write(event_fd, &data, sizeof (data)) != sizeof (data)) {
                err(1, "write to event_fd");
        }
 }

 void read_from_event_fd(int event_fd) {
        static uint64_t want_data = 1;
        uint64_t have_data;

        if (read(event_fd, &have_data, sizeof (have_data)) != sizeof (have_data)) {
                err(1, "read from event_fd");
        }

        if (want_data != have_data) {
                err(1, "read bad data from event_fd");
        }
 }

 void wait_for_epoll_events(int epoll_fd) {
        int nevents = 50;
        int timeout = -1;
        struct epoll_event ep_events[nevents];
        int num_events;

        num_events = epoll_wait(epoll_fd, ep_events, nevents, timeout);

        if (num_events != 1) {
                err(1, "epoll_wait returned bad value: %d", num_events);
        }
 }

 void *thread1_worker(void *ptr) {
        struct context *ctx = ptr;

        for (int i = 0; i < NUM_ITERATIONS; i++) {
                pthread_mutex_lock(ctx->lock);

                write_to_event_fd(ctx->event_fd);
                printf("[thread1]: (%d) wrote data to event_fd\n", i);

                pthread_cond_wait(ctx->cond, ctx->lock);
                pthread_mutex_unlock(ctx->lock);
        }

        printf("[thread1]: finished\n");
 }

 void *thread2_worker(void *ptr) {
        struct context *ctx = ptr;

        for (int i = 0; i < NUM_ITERATIONS; i++) {
                wait_for_epoll_events(ctx->epoll_fd);
                printf("[thread2]: (%d) got epoll events\n", i);

                pthread_mutex_lock(ctx->lock);

                if (READ_EVENT_FD) {
                        read_from_event_fd(ctx->event_fd);
                        printf("[thread2]: (%d) read from event_fd\n", i);
                }

                pthread_cond_signal(ctx->cond);
                pthread_mutex_unlock(ctx->lock);
        }

        printf("[thread2]: finished\n");
 }

 int main(int arg, char **argv) {
        int ret;
        struct epoll_event ev;
        pthread_t thread1, thread2;
        pthread_mutex_t lock;
        pthread_cond_t cond;

        printf("iterations: %d\n", NUM_ITERATIONS);
        printf("read-after-write: %s\n", READ_EVENT_FD ? "true" : "false");

        // create mutex
        if (pthread_mutex_init(&lock, NULL) != 0) {
                err(1, "pthread_mutex_init");
        }

        // create cond variable
        if (pthread_cond_init(&cond, NULL) != 0) {
                err(1, "pthread_cond_init");
        }

        // create epoll handle
        int epoll_fd = epoll_create1(EPOLL_CLOEXEC);
        if (epoll_fd == -1) {
                err(1, "epoll_create1");
        }

        // create eventfd handle
        int event_fd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
        if (event_fd == -1) {
                err(1, "eventfd create");
        }

        // add eventfd handle to epoll event watcher
        ev.events = EPOLLIN | EPOLLRDHUP | EPOLLET;
        ev.data.ptr = NULL;
        ret = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, event_fd, &ev);
        if (ret == -1) {
                err(1, "epoll_ctl (add eventfd to epoll)");
        }

        struct context ctx = {
                epoll_fd,
                event_fd,
                &lock,
                &cond
        };

        // create threads to test communication
        pthread_create(&thread1, NULL, thread1_worker, &ctx);
        pthread_create(&thread2, NULL, thread2_worker, &ctx);

        pthread_join(thread1, NULL);
        pthread_join(thread2, NULL);

        return 0;
 }
	/*
	* Simple test program to exercise possible LX bug wrt epoll/event2fd.
	*
	* This program works be creating an epoll fd and eventfd fd. The eventfd fd is
	* registered to be watched by the epoll fd, and then 2 threads are spun up to
	* communicate with eachother via the eventfd.
	*
	* Thread1:
	* - writes to the event fd
	* - waits for a condition signal to continue
	* - loop
	* Thread2:
	* - waits for epoll events on the event fd
	* - (optional) reads the data from the event fd
	* - signals thread 1 to wakeup
	* - loops
	*
	* The optional step of reading the data in Thread2 is what can trigger the bug.
	* If the data is read, this program works without issue. If the data is not
	* read, epoll will only generate an event for the first write to the eventfd,
	* but not subsequent writes.
	*
	* There are 2 #define's that can modify this programs behavior
	* - NUM_ITERATIONS - number of times to loop for each thread, should be >=2
	* to exercise this bug.
	* - READ_EVENT_FD - whether to read the data in Thread2 (the optional step),
	* setting this to `false` will exercise the bug and at this
	* current moment result in a hang after the 2nd call to
	* write.
	*
	* More info on bug origin:
	* https://gist.github.com/bahamas10/497acef27b8519e824e90d9eb648a6f6
	*
	* Author: Dave Eddy <[email protected]>
	* Date: March 19, 2022
	* License: MIT
	*/

	#include <err.h>
	#include <pthread.h>
	#include <stdbool.h>
	#include <stdio.h>
	#include <unistd.h>
	#include <sys/epoll.h>
	#include <sys/eventfd.h>

	#define READ_EVENT_FD false
	#define NUM_ITERATIONS 5

	// shared with threads
	struct context {
	int epoll_fd;
	int event_fd;
	pthread_mutex_t *lock;
	pthread_cond_t *cond;
	};

	void write_to_event_fd(int event_fd) {
	static uint64_t data = 1;

	if (write(event_fd, &data, sizeof (data)) != sizeof (data)) {
	err(1, "write to event_fd");
	}
	}

	void read_from_event_fd(int event_fd) {
	static uint64_t want_data = 1;
	uint64_t have_data;

	if (read(event_fd, &have_data, sizeof (have_data)) != sizeof (have_data)) {
	err(1, "read from event_fd");
	}

	if (want_data != have_data) {
	err(1, "read bad data from event_fd");
	}
	}

	void wait_for_epoll_events(int epoll_fd) {
	int nevents = 50;
	int timeout = -1;
	struct epoll_event ep_events[nevents];
	int num_events;

	num_events = epoll_wait(epoll_fd, ep_events, nevents, timeout);

	if (num_events != 1) {
	err(1, "epoll_wait returned bad value: %d", num_events);
	}
	}

	void thread1_worker(void ptr) {
	struct context *ctx = ptr;

	for (int i = 0; i < NUM_ITERATIONS; i++) {
	pthread_mutex_lock(ctx->lock);

	write_to_event_fd(ctx->event_fd);
	printf("[thread1]: (%d) wrote data to event_fd\n", i);

	pthread_cond_wait(ctx->cond, ctx->lock);
	pthread_mutex_unlock(ctx->lock);
	}

	printf("[thread1]: finished\n");
	}

	void thread2_worker(void ptr) {
	struct context *ctx = ptr;

	for (int i = 0; i < NUM_ITERATIONS; i++) {
	wait_for_epoll_events(ctx->epoll_fd);
	printf("[thread2]: (%d) got epoll events\n", i);

	pthread_mutex_lock(ctx->lock);

	if (READ_EVENT_FD) {
	read_from_event_fd(ctx->event_fd);
	printf("[thread2]: (%d) read from event_fd\n", i);
	}

	pthread_cond_signal(ctx->cond);
	pthread_mutex_unlock(ctx->lock);
	}

	printf("[thread2]: finished\n");
	}

	int main(int arg, char **argv) {
	int ret;
	struct epoll_event ev;
	pthread_t thread1, thread2;
	pthread_mutex_t lock;
	pthread_cond_t cond;

	printf("iterations: %d\n", NUM_ITERATIONS);
	printf("read-after-write: %s\n", READ_EVENT_FD ? "true" : "false");

	// create mutex
	if (pthread_mutex_init(&lock, NULL) != 0) {
	err(1, "pthread_mutex_init");
	}

	// create cond variable
	if (pthread_cond_init(&cond, NULL) != 0) {
	err(1, "pthread_cond_init");
	}

	// create epoll handle
	int epoll_fd = epoll_create1(EPOLL_CLOEXEC);
	if (epoll_fd == -1) {
	err(1, "epoll_create1");
	}

	// create eventfd handle
	int event_fd = eventfd(0, EFD_CLOEXEC \| EFD_NONBLOCK);
	if (event_fd == -1) {
	err(1, "eventfd create");
	}

	// add eventfd handle to epoll event watcher
	ev.events = EPOLLIN \| EPOLLRDHUP \| EPOLLET;
	ev.data.ptr = NULL;
	ret = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, event_fd, &ev);
	if (ret == -1) {
	err(1, "epoll_ctl (add eventfd to epoll)");
	}

	struct context ctx = {
	epoll_fd,
	event_fd,
	&lock,
	&cond
	};

	// create threads to test communication
	pthread_create(&thread1, NULL, thread1_worker, &ctx);
	pthread_create(&thread2, NULL, thread2_worker, &ctx);

	pthread_join(thread1, NULL);
	pthread_join(thread2, NULL);

	return 0;
	}