Nasty things to do when creating a MT application without any support library like pthread

clone.c

// compile with gcc -O0 (no optimizations), 
// otherwise code is broken with CLONE_THREAD being enabled

#define _GNU_SOURCE

#include <stdio.h>
#include <string.h>
#include <sched.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/wait.h>
#include <sys/syscall.h>
#include <sys/mman.h>

//syscall instruction affects the following regs, see manuals
#define __X86_SYSCALL_INSTR_CLOBBERS "rcx", "r11", "cc"

#define __base_CLONE_THREAD_FLAGS \
	CLONE_SIGHAND | \
	CLONE_VM | \
	CLONE_FS | \
	CLONE_FILES | \
	CLONE_PARENT_SETTID

#ifndef _MORE_CTF
#define _MORE_CTF | CLONE_THREAD | CLONE_CHILD_CLEARTID
#endif

#define CLONE_THREAD_FLAGS (__base_CLONE_THREAD_FLAGS _MORE_CTF)

#define ctf_has(feature) (CLONE_THREAD_FLAGS & feature)

#define REQUIRES_BUSYWAIT \
	ctf_has(CLONE_THREAD) \
	&& \
	!ctf_has(CLONE_CHILD_CLEARTID)

//show config infos
#ifdef _USE_SYSCALL_LIB
#	warning SYSCALL METHOD: using syscall() from glibc
#else
#	warning SYSCALL METHOD: issuing raw syscall instruction
#endif

#if ctf_has(CLONE_CHILD_CLEARTID)
#	warning CLEARTID SUPPORT: enabled
#else
#	warning CLEARTID SUPPORT: disabled
#endif

#if ctf_has(CLONE_THREAD)
#	warning CLONE_THREAD: enabled 
#else
#	warning CLONE_THREAD: disabled (waitpid needed)
#endif

#if REQUIRES_BUSYWAIT
#	warning WAITING METHOD: busy-wait
#else
#	if ctf_has(CLONE_CHILD_CLEARTID) && ctf_has(CLONE_THREAD)
#		warning WAITING METHOD: futex
#	elif ctf_has(CLONE_CHILD_CLEARTID) && !ctf_has(CLONE_THREAD)
#		warning WAITING METHOD: futex + waitpid with WNOHANG
#	else
#		warning WAITING METHOD: waitpid
#	endif
#endif

#if ctf_has(CLONE_THREAD) || ctf_has(CLONE_CHILD_CLEARTID)
#	define full_membar() \
		__asm__ __volatile__("mfence;" ::: "memory")
#endif

#if !ctf_has(CLONE_THREAD)
#	include <errno.h>
#endif

#if ctf_has(CLONE_CHILD_CLEARTID)
#	include <linux/futex.h>
#	define futex_wait(uaddr, val) \
        	syscall(SYS_futex, (uaddr), FUTEX_WAIT, (val), NULL, NULL, 0)
#endif

#define ull(x) ((unsigned long long)x)

#ifdef _USE_SYSCALL_LIB
#	define klone(_flags, _stack, _parent_tid, _child_tid, _tid, ___kloned_label___) \
		({ \
		 	int retval; \
		 	__asm__ __volatile__( \
		 		"movq %0, %%rdi\r\n" \
		 		"movq %1, %%rsi\r\n" \
		 		"movq %2, %%rdx\r\n" \
		 		"movq %3, %%rcx\r\n" \
		 		"movq %4, %%r8\r\n" \
		 		"movq %5, %%r9\r\n" \
		 		"call syscall\r\r" \
		 			:: \
		 				"i"(SYS_clone), \
		 				"r"(ull(_flags)), \
		 				"r"(ull(_stack)), \
		 				"r"(ull(_parent_tid)), \
		 				"r"(ull(_child_tid)), \
		 				"r"(ull(_tid)) \
		 			: \
		 				"%rdi", \
		 				"%rsi", \
		 				"%rdx", \
		 				"%rcx", \
		 				"%r8" \
		 	); \
		 	\
			___kloned_label___: \
			\
			__asm__ __volatile__( \
		 		"movl %%eax, %0\r\n" \
		 			: \
		 				"=m"(retval) \
		 			:: \
		 				"%eax" \
		 	); \
		 	retval; \
		 })
#else
#	define klone(_flags, _stack, _parent_tid, _child_tid, _tid, __unused__) \
		({ \
    		int retval; \
			__asm__ __volatile__( \
				"movq %1, %%rax\r\n" \
				\
				"movq %2, %%rdi\r\n" \
				"movq %3, %%rsi\r\n" \
				"movq %4, %%rdx\r\n" \
				"movq %5, %%r10\r\n" \
				"movq %6, %%r8\r\n" \
				"syscall\r\n" \
            	\
            	"movl %%eax, %0\r\n" \
					: \
                    	"=m"(retval) \
					: \
						"i"(SYS_clone), \
						"r"(ull(_flags)), \
						"r"(ull(_stack)), \
						"r"(ull(_parent_tid)), \
						"r"(ull(_child_tid)), \
						"r"(ull(_tid)) \
					: \
						"%rax", \
						"%rdi", \
						"%rsi", \
						"%rdx", \
						"%r10", \
						"%r8", \
						"%eax", \
						__X86_SYSCALL_INSTR_CLOBBERS \
			); \
    		retval; \
		})
#endif

// libc exit()s may call exit_group instead!!!
#define raw_thread_exit(code) \
	do { \
		__asm__ __volatile__( \
				"movq $60, %%rax\r\n" \
				"movl %0, %%edi\r\n" \
				"syscall\r\n" \
					:: \
						"r"(code) \
					: \
						"%rax", \
						"%edi", \
						__X86_SYSCALL_INSTR_CLOBBERS \
		); \
	} while(0)

typedef void(*thread_fn)(void*);

#define DEFAULT_STACK_SIZE (1024 * 1024)

struct thread_struct {
	char* stack_area;
	size_t stack_size;
	pid_t child_tid;
#if ctf_has(CLONE_THREAD) && !ctf_has(CLONE_CHILD_CLEARTID)
	int has_done;
#elif ctf_has(CLONE_CHILD_CLEARTID)
	int* cleartid;
#endif
};

#define THREAD_CREATE_MMAP ((struct thread_struct*) -1)
#define THREAD_CREATE_CLONE ((struct thread_struct*) -2)
#define THREAD_CREATE_MALLOC ((struct thread_struct*) -3)

#if REQUIRES_BUSYWAIT
#	define DEFINE_NEW_THREAD_WRAPPER(arg1, arg2, arg3, arg4) \
		static void new_thread_wrapper(struct thread_struct **arg1, int *arg2, thread_fn arg3, void* arg4)
#	define CALL_NEW_THREAD_WRAPPER(arg1, arg2, arg3, arg4) \
		new_thread_wrapper((arg1), (arg2), (arg3), (arg4))
#else
#	define DEFINE_NEW_THREAD_WRAPPER(arg1, arg2, arg3, arg4) \
		static void new_thread_wrapper(thread_fn arg3, void *arg4)
#	define CALL_NEW_THREAD_WRAPPER(arg1, arg2, arg3, arg4) \
		new_thread_wrapper((arg3), (arg4))
#endif

//this is subtle: require a new base pointer
DEFINE_NEW_THREAD_WRAPPER(new_thr, copied, fn, arg) {

#if REQUIRES_BUSYWAIT

	while(!*new_thr)
		;

	struct thread_struct *cpy_thr = *new_thr;

	full_membar();
	*copied = 1;
#endif

	fn(arg);

#if REQUIRES_BUSYWAIT

	full_membar();
	cpy_thr->has_done = 1;
#endif

	raw_thread_exit(0);
}

static inline struct thread_struct *__thread_struct_alloc() {
	return (struct thread_struct*) malloc(sizeof(struct thread_struct));
}

static inline void __thread_struct_free(struct thread_struct *thr) {
	free(thr);
}

static struct thread_struct* __thread_create(thread_fn fn, void* arg, size_t stack_size) {
	struct thread_struct *new_thr = NULL;

#if REQUIRES_BUSYWAIT
	int copied = 0;
#endif

	char* stack = (char*) mmap(
			NULL, 
			stack_size, 
			PROT_READ | PROT_WRITE, 
			MAP_PRIVATE | MAP_ANONYMOUS, 
			-1, 0);

	if(stack == MAP_FAILED) {
		return THREAD_CREATE_MMAP;
	}

	char* top_of_stack = stack + stack_size;
#ifdef _USE_SYSCALL_LIB 
#	define __copy_ith_byte(i) \
		*((char*) top_of_stack + (i - 8)) = \
			(char) ((ull(&&__new_thread_ret) >> (i * 8)) & 0xff)

	for(int i = 0; i < 8; i++) {
		__copy_ith_byte(i);
	}

#	undef __copy_ith_byte

	top_of_stack -= 8;
#endif

	int *cleartid = NULL;

#if ctf_has(CLONE_CHILD_CLEARTID)
	cleartid = (int*) malloc(sizeof(int));
	if(!cleartid) {
		munmap(stack, stack_size);
		return THREAD_CREATE_MALLOC;
	}

	*cleartid = 1;
	full_membar();
#endif

	pid_t child_tid;
	int rv = klone(CLONE_THREAD_FLAGS, top_of_stack, &child_tid, cleartid, 0, __new_thread_ret);

	if(rv == -1) {

#if ctf_has(CLONE_CHILD_CLEARTID)
		free(cleartid);
#endif

		munmap(stack, stack_size);
		return THREAD_CREATE_CLONE;
	} else if(!rv) {
		CALL_NEW_THREAD_WRAPPER(&new_thr, &copied, fn, arg);
	}

	//this is needed for synchronization purposes
#if REQUIRES_BUSYWAIT
	struct thread_struct* _tmp_thr = __thread_struct_alloc();
	if(!_tmp_thr) {
		munmap(stack, stack_size);
		return THREAD_CREATE_MALLOC;
	}

	memset(_tmp_thr, 0, sizeof(struct thread_struct));

	full_membar();
	new_thr = _tmp_thr;
#else
	new_thr = __thread_struct_alloc();

	if(!new_thr) {

#	if ctf_has(CLONE_CHILD_CLEARTID)
		free(cleartid);
#	endif

		munmap(stack, stack_size);
		return THREAD_CREATE_MALLOC;
	}
#endif

	new_thr->stack_area = stack;
	new_thr->stack_size = stack_size;
	new_thr->child_tid = child_tid;

#if ctf_has(CLONE_CHILD_CLEARTID)
	new_thr->cleartid = cleartid;
#endif

#if REQUIRES_BUSYWAIT
	while(!copied)
		;
#endif

	return new_thr;
}

static struct thread_struct* thread_create(thread_fn fn, void* arg) {
	return __thread_create(fn, arg, DEFAULT_STACK_SIZE);
}

#if ctf_has(CLONE_CHILD_CLEARTID)
#	define __do_futex_wait(fut) \
		do { \
			futex_wait((fut), 1); \
			free((fut)); \
			(fut) = NULL; \
		} while(0)
#endif

static void thread_join(struct thread_struct *thr) {
#if !ctf_has(CLONE_THREAD)
	int wpid_flags = __WCLONE;

#	if ctf_has(CLONE_CHILD_CLEARTID)
	wpid_flags |= WNOHANG;
	__do_futex_wait(thr->cleartid);
#	endif
	
	if(waitpid(thr->child_tid, NULL, wpid_flags) < 0) {
		fprintf(stderr, "waitpid (%d): %s\n", thr->child_tid, strerror(errno));
		abort();
	}
#else
#	if !ctf_has(CLONE_CHILD_CLEARTID)
	while(!thr->has_done)
		;
#	else
	__do_futex_wait(thr->cleartid);
#	endif
#endif

	munmap(thr->stack_area, thr->stack_size);
	__thread_struct_free(thr);
}

#if ctf_has(CLONE_CHILD_CLEARTID)
#	undef __do_futex_wait
#endif

#define boolstr(expr) ((expr) ? "true" : "false")

static void hello_fn(void* arg) {
	char buf[8192];
	memset(buf, 0, 8192);
	snprintf(buf, 8192, "thread %d of process %d, about to sleep: %s\n", 
			gettid(), getpid(), boolstr(arg != (void*)-1));

	write(STDOUT_FILENO, buf, strlen(buf));

	if(arg != (void*)-1) {
		sleep(1);
	}
}

#undef boolstr

int main() {
	hello_fn((void*)-1);

	struct thread_struct *thr1 = thread_create(hello_fn, NULL);
	struct thread_struct *thr2 = thread_create(hello_fn, NULL);

	thread_join(thr1);
	thread_join(thr2);

	return 0;
}

test.sh

#!/bin/bash

SRC=clone.c
BASE_CMD="gcc -O0 $SRC -o clone"

banner() {
	echo ""
	echo " ---> Running config <--- "
	echo "     Description: $@      "
	echo ""
}

run() {
	./clone || exit 1 
}

# this doesn't work and idk why
#banner "syscall lib"
#$BASE_CMD -D_USE_SYSCALL_LIB -D_MORE_CTF=""
#run
#

banner "syscall lib + thread/LWP numbering"
$BASE_CMD -D_USE_SYSCALL_LIB -D_MORE_CTF="|CLONE_THREAD"
run

banner "syscall lib + cleartid/futex"
$BASE_CMD -D_USE_SYSCALL_LIB -D_MORE_CTF="|CLONE_CHILD_CLEARTID"
run

banner "syscall lib + thread/LWP numbering + cleartid/futex"
$BASE_CMD -D_USE_SYSCALL_LIB -D_MORE_CTF="|CLONE_THREAD|CLONE_CHILD_CLEARTID"
run

banner 
$BASE_CMD -D_MORE_CTF=""
run

banner "thread/LWP numbering"
$BASE_CMD -D_MORE_CTF="|CLONE_THREAD"
run

banner "cleartid/futex"
$BASE_CMD -D_MORE_CTF="|CLONE_CHILD_CLEARTID"
run

banner "thread/LWP numbering + cleartid/futex"
$BASE_CMD -D_MORE_CTF="|CLONE_THREAD|CLONE_CHILD_CLEARTID"
run