Extreme GNU/Weeb HTTP Server

main.c

// SPDX-License-Identifier: GPL-2.0-only
/**
 * Copyright (C) 2022  Ammar Faizi <[email protected]>
 *
 * This is the Extreme GNU/Weeb HTTP Server. I wrote this program
 * because of a challenge from Alviro. The challenge is to create
 * a web server application *without including any library*. The
 * web server has to show the directory list and handle download
 * file. It must also be able to handle 10K concurrent requests.
 *
 * Link: https://t.me/GNUWeeb/681387
 *
 * How to run this:
 *   gcc -Wall -Wextra -ffreestanding -nostdlib -nostartfiles -static -ggdb3 -Os main.c -o main;
 *   ./main;
 *   ab -n 50000 -c 10000 http://127.0.0.1:9999/
 */
#define NR_WORKERS	8
#define NR_MAX_EVENTS	512
#define NR_CLIENTS	300000
#define BIND_PORT	9999
#define CLIENT_BUFFER	1024

__asm__ (
".global _start\n"
".section .text\n"
"_start:\n\t"
	"xorl	%ebp, %ebp\n\t"
	"andq	$-16, %rsp\n\t"
	"subq	$8, %rsp\n\t"
	"jmp	__start"
);

typedef unsigned long long u64;
typedef unsigned int u32;
typedef unsigned short u16;
typedef unsigned char u8;
typedef unsigned long size_t;
typedef long ssize_t;
typedef long off_t;
typedef u64 __aligned_u64;
typedef u64 rlim_t;
typedef long intptr_t;
typedef unsigned long uintptr_t;
typedef _Bool bool;
typedef u32 uint32_t;
typedef u64 uint64_t;
typedef u16 uint16_t;
typedef u8 uint8_t;
typedef unsigned int socklen_t;
typedef long pid_t;
typedef unsigned long __kernel_ulong_t;
typedef long __kernel_long_t;
typedef long ino64_t;
typedef long off64_t;
typedef long ptrdiff_t;

/*
 * Ref: https://github.com/torvalds/linux/blob/v6.1/arch/x86/include/uapi/asm/stat.h#L83-L104
 */
struct stat {
	__kernel_ulong_t	st_dev;
	__kernel_ulong_t	st_ino;
	__kernel_ulong_t	st_nlink;

	unsigned int		st_mode;
	unsigned int		st_uid;
	unsigned int		st_gid;
	unsigned int		__pad0;
	__kernel_ulong_t	st_rdev;
	__kernel_long_t		st_size;
	__kernel_long_t		st_blksize;
	__kernel_long_t		st_blocks;

	__kernel_ulong_t	st_atime;
	__kernel_ulong_t	st_atime_nsec;
	__kernel_ulong_t	st_mtime;
	__kernel_ulong_t	st_mtime_nsec;
	__kernel_ulong_t	st_ctime;
	__kernel_ulong_t	st_ctime_nsec;
	__kernel_long_t		__unused[3];
};

struct linux_dirent64 {
	ino64_t		d_ino;
	off64_t		d_off;
	unsigned short	d_reclen;
	unsigned char	d_type;
	char		d_name[];
};

struct rlimit {
	rlim_t rlim_cur;  /* Soft limit */
	rlim_t rlim_max;  /* Hard limit (ceiling for rlim_cur) */
};

/*
 * Ref: https://github.com/torvalds/linux/blob/v6.1/include/uapi/asm-generic/signal-defs.h#L82-L83
 */
typedef void __signalfn_t(int);
typedef __signalfn_t *__sighandler_t;
#define SIG_IGN ((__sighandler_t)1)

/*
 * Ref: https://github.com/torvalds/linux/blob/v6.1/arch/x86/include/uapi/asm/signal.h#L79-L87
 */
#define NSIG		32
typedef unsigned long sigset_t;
#define SIGPIPE	13
struct siginfo;
struct sigaction {
	union {
		__sighandler_t _sa_handler;
		void (*_sa_sigaction)(int, struct siginfo *, void *);
	} _u;
	sigset_t sa_mask;
	unsigned long sa_flags;
	void (*sa_restorer)(void);
};

/*
 * Ref: https://github.com/torvalds/linux/blob/v6.1/include/uapi/linux/sched.h#L92-L104
 */
struct clone_args {
	__aligned_u64 flags;
	__aligned_u64 pidfd;
	__aligned_u64 child_tid;
	__aligned_u64 parent_tid;
	__aligned_u64 exit_signal;
	__aligned_u64 stack;
	__aligned_u64 stack_size;
	__aligned_u64 tls;
	__aligned_u64 set_tid;
	__aligned_u64 set_tid_size;
	__aligned_u64 cgroup;
};

typedef union epoll_data {
	void		*ptr;
	int		fd;
	uint32_t	u32;
	uint64_t	u64;
} epoll_data_t;

struct epoll_event {
	uint32_t	events;
	epoll_data_t	data;
} __attribute__((__packed__));

struct in_addr {
	unsigned long		s_addr;
};

struct sockaddr_in {
	short		sin_family;
	unsigned short	sin_port;
	struct in_addr	sin_addr;
	char		sin_zero[8];
};

struct sockaddr {
	short		sin_family;
	char		pad[sizeof(struct sockaddr_in)];
};

typedef union mutex {
	unsigned u;
	struct {
		unsigned char locked;
		unsigned char contended;
	} b;
} mutex_t;

struct server_ctx;

struct worker {
	struct server_ctx	*sctx;
	int			epl_fd;
	u32			idx;
	u32			acc_cycle;
	pid_t			pid;
	struct epoll_event	events[NR_MAX_EVENTS];
};

struct client {
	int			fd;
	u32			idx;
	struct sockaddr_in	addr4;
	char			buf[CLIENT_BUFFER];
	size_t			buf_len;
	char			*uri;

	struct {
		char		*map;
		size_t		pos;
		size_t		len;
	} pfile;
};

struct stack {
	uint32_t		*arr;
	uint32_t		max;
	uint32_t		sp;
	mutex_t			mutex;
};

struct server_ctx {
	int			tcp_fd;

	u32			nr_workers;
	u32			pos_worker;
	struct worker		*workers;
	mutex_t			worker_mutex;

	struct stack		client_stk;
	u32			nr_clients;
	struct client		clients[NR_CLIENTS];
};

struct simple_heap {
	size_t			len;
	char			user_data[] __attribute__((__aligned__));
};

/*
 * Ref: https://github.com/torvalds/linux/blob/v6.1/include/uapi/linux/sched.h#L10-L34
 */
#define CSIGNAL			0x000000ff
#define CLONE_VM		0x00000100
#define CLONE_FS		0x00000200
#define CLONE_FILES		0x00000400
#define CLONE_SIGHAND		0x00000800
#define CLONE_PIDFD		0x00001000
#define CLONE_PTRACE		0x00002000
#define CLONE_VFORK		0x00004000
#define CLONE_PARENT		0x00008000
#define CLONE_THREAD		0x00010000
#define CLONE_NEWNS		0x00020000
#define CLONE_SYSVSEM		0x00040000
#define CLONE_SETTLS		0x00080000
#define CLONE_PARENT_SETTID	0x00100000
#define CLONE_CHILD_CLEARTID	0x00200000
#define CLONE_DETACHED		0x00400000
#define CLONE_UNTRACED		0x00800000
#define CLONE_CHILD_SETTID	0x01000000
#define CLONE_NEWCGROUP		0x02000000
#define CLONE_NEWUTS		0x04000000
#define CLONE_NEWIPC		0x08000000
#define CLONE_NEWUSER		0x10000000
#define CLONE_NEWPID		0x20000000
#define CLONE_NEWNET		0x40000000
#define CLONE_IO		0x80000000

/*
 * Ref: https://github.com/torvalds/linux/blob/v6.1/include/uapi/linux/futex.h#L11-L44
 */
#define FUTEX_WAIT		0
#define FUTEX_WAKE		1
#define FUTEX_FD		2
#define FUTEX_REQUEUE		3
#define FUTEX_CMP_REQUEUE	4
#define FUTEX_WAKE_OP		5
#define FUTEX_LOCK_PI		6
#define FUTEX_UNLOCK_PI		7
#define FUTEX_TRYLOCK_PI	8
#define FUTEX_WAIT_BITSET	9
#define FUTEX_WAKE_BITSET	10
#define FUTEX_WAIT_REQUEUE_PI	11
#define FUTEX_CMP_REQUEUE_PI	12
#define FUTEX_LOCK_PI2		13

#define FUTEX_PRIVATE_FLAG	128
#define FUTEX_CLOCK_REALTIME	256
#define FUTEX_CMD_MASK		~(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME)

#define FUTEX_WAIT_PRIVATE	(FUTEX_WAIT | FUTEX_PRIVATE_FLAG)
#define FUTEX_WAKE_PRIVATE	(FUTEX_WAKE | FUTEX_PRIVATE_FLAG)
#define FUTEX_REQUEUE_PRIVATE	(FUTEX_REQUEUE | FUTEX_PRIVATE_FLAG)
#define FUTEX_CMP_REQUEUE_PRIVATE (FUTEX_CMP_REQUEUE | FUTEX_PRIVATE_FLAG)
#define FUTEX_WAKE_OP_PRIVATE	(FUTEX_WAKE_OP | FUTEX_PRIVATE_FLAG)
#define FUTEX_LOCK_PI_PRIVATE	(FUTEX_LOCK_PI | FUTEX_PRIVATE_FLAG)
#define FUTEX_LOCK_PI2_PRIVATE	(FUTEX_LOCK_PI2 | FUTEX_PRIVATE_FLAG)
#define FUTEX_UNLOCK_PI_PRIVATE	(FUTEX_UNLOCK_PI | FUTEX_PRIVATE_FLAG)
#define FUTEX_TRYLOCK_PI_PRIVATE (FUTEX_TRYLOCK_PI | FUTEX_PRIVATE_FLAG)
#define FUTEX_WAIT_BITSET_PRIVATE	(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG)
#define FUTEX_WAKE_BITSET_PRIVATE	(FUTEX_WAKE_BITSET | FUTEX_PRIVATE_FLAG)
#define FUTEX_WAIT_REQUEUE_PI_PRIVATE	(FUTEX_WAIT_REQUEUE_PI | \
					 FUTEX_PRIVATE_FLAG)
#define FUTEX_CMP_REQUEUE_PI_PRIVATE	(FUTEX_CMP_REQUEUE_PI | \
					 FUTEX_PRIVATE_FLAG)

#define PROT_READ		0x1
#define PROT_WRITE		0x2
#define PROT_EXEC		0x4

#define MAP_SHARED		0x01
#define MAP_PRIVATE		0x02
#define MAP_SHARED_VALIDATE	0x03
#define MAP_ANONYMOUS		0x20
#define MAP_POPULATE		0x008000
#define MAP_NONBLOCK		0x010000
#define MAP_STACK		0x020000

#define SOCK_STREAM		1
#define SOCK_NONBLOCK		04000
#define AF_INET			2


#define EPOLL_CTL_ADD		1
#define EPOLL_CTL_DEL		2
#define EPOLL_CTL_MOD		3
#define EPOLLIN			0x00000001
#define EPOLLPRI		0x00000002
#define EPOLLOUT		0x00000004
#define EPOLLERR		0x00000008
#define EPOLLHUP		0x00000010
#define EPOLLNVAL		0x00000020
#define EPOLLRDNORM		0x00000040
#define EPOLLRDBAND		0x00000080
#define EPOLLWRNORM		0x00000100
#define EPOLLWRBAND		0x00000200
#define EPOLLMSG		0x00000400
#define EPOLLRDHUP		0x00002000
#define EPOLLEXCLUSIVE 		(1U << 28)
#define EPOLLWAKEUP 		(1U << 29)
#define EPOLLONESHOT 		(1U << 30)
#define EPOLLET 		(1U << 31)

#define EINTR			4
#define EAGAIN			11
#define ENOMEM			12
#define EINVAL			22
#define ENETDOWN		100

#define SOL_SOCKET	1
#define SO_REUSEADDR	2
#define SO_REUSEPORT	15

#define O_RDONLY        00000000

#define unlikely(COND)		__builtin_expect(!!(COND), 0)
#define likely(COND)		__builtin_expect(!!(COND), 1)
#define NULL			((void *)0)

#define S_IFMT  00170000
#define S_IFSOCK 0140000
#define S_IFLNK	 0120000
#define S_IFREG  0100000
#define S_IFBLK  0060000
#define S_IFDIR  0040000
#define S_IFCHR  0020000
#define S_IFIFO  0010000
#define S_ISUID  0004000
#define S_ISGID  0002000
#define S_ISVTX  0001000

#define S_ISLNK(m)	(((m) & S_IFMT) == S_IFLNK)
#define S_ISREG(m)	(((m) & S_IFMT) == S_IFREG)
#define S_ISDIR(m)	(((m) & S_IFMT) == S_IFDIR)
#define S_ISCHR(m)	(((m) & S_IFMT) == S_IFCHR)
#define S_ISBLK(m)	(((m) & S_IFMT) == S_IFBLK)
#define S_ISFIFO(m)	(((m) & S_IFMT) == S_IFIFO)
#define S_ISSOCK(m)	(((m) & S_IFMT) == S_IFSOCK)

#define S_IRWXU 00700
#define S_IRUSR 00400
#define S_IWUSR 00200
#define S_IXUSR 00100

#define S_IRWXG 00070
#define S_IRGRP 00040
#define S_IWGRP 00020
#define S_IXGRP 00010

#define S_IRWXO 00007
#define S_IROTH 00004
#define S_IWOTH 00002
#define S_IXOTH 00001


#define true 1
#define false 0
#define noreturn		__attribute__((__noreturn__))
#define noinline		__attribute__((__noinline__))
#define __used			__attribute__((__used__))
#define __maybe_unused		__attribute__((__unused__))
#define fallthrough		__attribute__((__fallthrough__))
#ifndef __always_inline
#define __always_inline		__attribute__((__always_inline__)) inline
#endif

#define STRL(STR)		STR, (sizeof(STR) - 1)

#ifndef offsetof
#define offsetof(TYPE, FIELD) ((size_t) &((TYPE *)0)->FIELD)
#endif

#ifndef container_of
#define container_of(PTR, TYPE, FIELD) ({			\
	__typeof__(((TYPE *)0)->FIELD) *__FIELD_PTR = (PTR);	\
	(TYPE *)((char *) __FIELD_PTR - offsetof(TYPE, FIELD));	\
})
#endif

#if defined __STDC_VERSION__ && __STDC_VERSION__ > 201710L
#define va_start(v, ...)	__builtin_va_start(v, 0)
#else
#define va_start(v,l)	__builtin_va_start(v,l)
#endif
#define va_end(v)	__builtin_va_end(v)
#define va_arg(v,l)	__builtin_va_arg(v,l)
#if !defined(__STRICT_ANSI__) || __STDC_VERSION__ + 0 >= 199900L \
    || __cplusplus + 0 >= 201103L
#define va_copy(d,s)	__builtin_va_copy(d,s)
#endif
#define __va_copy(d,s)	__builtin_va_copy(d,s)

#ifndef __GNUC_VA_LIST
#define __GNUC_VA_LIST
typedef __builtin_va_list __gnuc_va_list;
#endif

#ifndef __va_list__
typedef __gnuc_va_list va_list;
#endif /* not __va_list__ */

/**
 * Note for syscall registers usage (x86-64):
 *   - %rax is the syscall number.
 *   - %rax is also the return value.
 *   - %rdi is the 1st argument.
 *   - %rsi is the 2nd argument.
 *   - %rdx is the 3rd argument.
 *   - %r10 is the 4th argument (**yes it's %r10, not %rcx!**).
 *   - %r8  is the 5th argument.
 *   - %r9  is the 6th argument.
 *
 * `syscall` instruction will clobber %r11 and %rcx.
 *
 * After the syscall returns to userspace:
 *   - %r11 will contain %rflags.
 *   - %rcx will contain the return address.
 *
 * IOW, after the syscall returns to userspace:
 *   %r11 == %rflags and %rcx == %rip.
 */

#define __do_syscall0(NUM) ({			\
	long rax;				\
						\
	__asm__ volatile(			\
		"syscall"			\
		: "=a"(rax)	/* %rax */	\
		: "a"(NUM)	/* %rax */	\
		: "rcx", "r11", "memory"	\
	);					\
	rax;					\
})

#define __do_syscall1(NUM, ARG1) ({		\
	long rax;				\
						\
	__asm__ volatile(			\
		"syscall"			\
		: "=a"(rax)	/* %rax */	\
		: "a"((NUM)),	/* %rax */	\
		  "D"((ARG1))	/* %rdi */	\
		: "rcx", "r11", "memory"	\
	);					\
	rax;					\
})

#define __do_syscall2(NUM, ARG1, ARG2) ({	\
	long rax;				\
						\
	__asm__ volatile(			\
		"syscall"			\
		: "=a"(rax)	/* %rax */	\
		: "a"((NUM)),	/* %rax */	\
		  "D"((ARG1)),	/* %rdi */	\
		  "S"((ARG2))	/* %rsi */	\
		: "rcx", "r11", "memory"	\
	);					\
	rax;					\
})

#define __do_syscall3(NUM, ARG1, ARG2, ARG3) ({	\
	long rax;				\
						\
	__asm__ volatile(			\
		"syscall"			\
		: "=a"(rax)	/* %rax */	\
		: "a"((NUM)),	/* %rax */	\
		  "D"((ARG1)),	/* %rdi */	\
		  "S"((ARG2)),	/* %rsi */	\
		  "d"((ARG3))	/* %rdx */	\
		: "rcx", "r11", "memory"	\
	);					\
	rax;					\
})

#define __do_syscall4(NUM, ARG1, ARG2, ARG3, ARG4) ({			\
	long rax;							\
	register __typeof__(ARG4) __r10 __asm__("r10") = (ARG4);	\
									\
	__asm__ volatile(						\
		"syscall"						\
		: "=a"(rax)	/* %rax */				\
		: "a"((NUM)),	/* %rax */				\
		  "D"((ARG1)),	/* %rdi */				\
		  "S"((ARG2)),	/* %rsi */				\
		  "d"((ARG3)),	/* %rdx */				\
		  "r"(__r10)	/* %r10 */				\
		: "rcx", "r11", "memory"				\
	);								\
	rax;								\
})

#define __do_syscall5(NUM, ARG1, ARG2, ARG3, ARG4, ARG5) ({		\
	long rax;							\
	register __typeof__(ARG4) __r10 __asm__("r10") = (ARG4);	\
	register __typeof__(ARG5) __r8 __asm__("r8") = (ARG5);		\
									\
	__asm__ volatile(						\
		"syscall"						\
		: "=a"(rax)	/* %rax */				\
		: "a"((NUM)),	/* %rax */				\
		  "D"((ARG1)),	/* %rdi */				\
		  "S"((ARG2)),	/* %rsi */				\
		  "d"((ARG3)),	/* %rdx */				\
		  "r"(__r10),	/* %r10 */				\
		  "r"(__r8)	/* %r8 */				\
		: "rcx", "r11", "memory"				\
	);								\
	rax;								\
})

#define __do_syscall6(NUM, ARG1, ARG2, ARG3, ARG4, ARG5, ARG6) ({	\
	long rax;							\
	register __typeof__(ARG4) __r10 __asm__("r10") = (ARG4);	\
	register __typeof__(ARG5) __r8 __asm__("r8") = (ARG5);		\
	register __typeof__(ARG6) __r9 __asm__("r9") = (ARG6);		\
									\
	__asm__ volatile(						\
		"syscall"						\
		: "=a"(rax)	/* %rax */				\
		: "a"((NUM)),	/* %rax */				\
		  "D"((ARG1)),	/* %rdi */				\
		  "S"((ARG2)),	/* %rsi */				\
		  "d"((ARG3)),	/* %rdx */				\
		  "r"(__r10),	/* %r10 */				\
		  "r"(__r8),	/* %r8 */				\
		  "r"(__r9)	/* %r9 */				\
		: "rcx", "r11", "memory"				\
	);								\
	rax;								\
})

noreturn static inline void __sys_exit(int code)
{
	__do_syscall1(60, code);
	__builtin_unreachable();
}

noreturn static inline void __sys_exit_group(int code)
{
	__do_syscall1(231, code);
	__builtin_unreachable();
}

static inline int __sys_open(const char *pathname, int flags, int mode)
{
	return (int) __do_syscall3(2, pathname, flags, mode);
}

static inline ssize_t __sys_write(int fd, const void *buffer, size_t size)
{
	return (ssize_t) __do_syscall3(1, fd, buffer, size);
}

static inline ssize_t __sys_read(int fd, void *buffer, size_t size)
{
	return (ssize_t) __do_syscall3(0, fd, buffer, size);
}

static inline void *__sys_mmap(void *addr, size_t length, int prot, int flags,
			       int fd, off_t offset)
{
	return (void *) __do_syscall6(9, addr, length, prot, flags, fd, offset);
}

static inline int __sys_munmap(void *addr, size_t length)
{
	return (int) __do_syscall2(11, addr, length);
}

#if 0
static inline int __sys_getrlimit(int resource, struct rlimit *rlim)
{
	return (int) __do_syscall2(97, resource, rlim);
}

static inline int __sys_setrlimit(int resource, const struct rlimit *rlim)
{
	return (int) __do_syscall2(160, resource, rlim);
}
#endif

static inline int __sys_close(int fd)
{
	return (int) __do_syscall1(3, fd);
}

static inline int __sys_socket(int domain, int type, int protocol)
{
	return (int) __do_syscall3(41, domain, type, protocol);
}

static inline int __sys_accept4(int fd, struct sockaddr *addr,
				socklen_t *addrlen, int flags)
{
	return (int) __do_syscall4(288, fd, addr, addrlen, flags);
}

static inline int __sys_bind(int sockfd, const struct sockaddr *addr,
		       socklen_t addrlen)
{
	return (int) __do_syscall3(49, sockfd, addr, addrlen);
}

static inline int __sys_listen(int sockfd, int backlog)
{
	return (int) __do_syscall2(50, sockfd, backlog);
}

static inline int __sys_epoll_create(int size)
{
	return (int) __do_syscall1(213, size);
}

static inline int __sys_epoll_wait(int epfd, struct epoll_event *events,
				   int maxevents, int timeout)
{
	return (int) __do_syscall4(232, epfd, events, maxevents, timeout);
}

static inline int __sys_epoll_ctl(int epfd, int op, int fd,
				  struct epoll_event *event)
{
	return (int) __do_syscall4(233, epfd, op, fd, event);
}

struct timespec;

static inline long __sys_futex(uint32_t *uaddr, int futex_op, uint32_t val,
			       const struct timespec *timeout, uint32_t *uaddr2,
			       uint32_t val3)
{
	return (long) __do_syscall6(202, uaddr, futex_op, val, timeout, uaddr2,
				    val3);
}

static inline int __sys_setsockopt(int sockfd, int level, int optname,
				   const void *optval, socklen_t optlen)
{
	return (int) __do_syscall5(54, sockfd, level, optname, optval, optlen);
}

static inline int __sys_fstat(int fd, struct stat *statbuf)
{
	return (int) __do_syscall2(5, fd, statbuf);
}

static inline int __sys_stat(const char *file, struct stat *statbuf)
{
	return (int) __do_syscall2(4, file, statbuf);
}

static ssize_t __sys_getdents64(int fd, void *dirp, size_t count)
{
	return (int) __do_syscall3(217, fd, dirp, count);
}

static inline int __sys_rt_sigaction(int sig, const struct sigaction *act,
				     struct sigaction *oact,
				     size_t sigsetsize)
{
	return (int) __do_syscall4(13, sig, act, oact, sigsetsize);
}

__maybe_unused
static inline void *ERR_PTR(intptr_t n)
{
	return (void *) n;
}

static inline int PTR_ERR(const void *ptr)
{
	return (int) (intptr_t) ptr;
}

static inline bool IS_ERR(const void *ptr)
{
	return unlikely((uintptr_t) ptr >= (uintptr_t) -4095UL);
}

extern noreturn void __start_thread(void *(*target_func)(void *arg), void *arg);

noreturn void __start_thread(void *(*target_func)(void *arg), void *arg)
{
	target_func(arg);
	__sys_exit(0);
}

static int mutex_init(mutex_t *m)
{
	m->u = 0;
	return 0;
}

static __always_inline void barrier(void)
{
	__asm__ volatile ("mfence" ::: "memory");
}

static __always_inline void cpu_relax(void)
{
	__asm__ volatile ("pause" ::: "memory");
}

static __always_inline u8 xchg_8(u8 *ptr, u8 x)
{
	return __atomic_exchange_n(ptr, x, __ATOMIC_SEQ_CST);
}

static __always_inline u32 xchg_32(u32 *ptr, u32 x)
{
	return __atomic_exchange_n(ptr, x, __ATOMIC_SEQ_CST);
}

static __always_inline u32 cmpxchg_32(uint32_t *ptr, u32 old, u32 new)
{
	u32 ret;

	__asm__ volatile (
		"cmpxchgl %2, %1"
		: "=a"(ret), "+m"(*ptr)
		: "q"(new), "0"(old)
		: "memory"
	);
	return ret;
}

static int puts(const char *p);

noinline static int mutex_lock(mutex_t *m)
{
	int i;

	/* Try to grab lock */
	for (i = 0; i < 100; i++) {
		if (!xchg_8(&m->b.locked, 1))
			return 0;

		cpu_relax();
	}

	/* Have to sleep */
	while (xchg_32(&m->u, 257) & 1)
		__sys_futex((void *)m, FUTEX_WAIT_PRIVATE, 257, NULL, NULL, 0);

	return 0;
}

noinline static int mutex_unlock(mutex_t *m)
{
	int i;

	/* Locked and not contended */
	if ((m->u == 1) && (cmpxchg_32(&m->u, 1, 0) == 1))
		return 0;

	/* Unlock */
	m->b.locked = 0;
	barrier();

	/* Spin and hope someone takes the lock */
	for (i = 0; i < 200; i++) {
		if (m->b.locked)
			return 0;

		cpu_relax();
	}

	/* We need to wake someone up */
	m->b.contended = 0;
	__sys_futex((void *)m, FUTEX_WAKE_PRIVATE, 1, NULL, NULL, 0);
	return 0;
}

static inline int do_clone(struct clone_args *args, size_t size,
			   void *(*func)(void *data), void *data)
{
	pid_t rax = 435;

	__asm__ volatile (
		"syscall\n\t"
		"testq	%%rax, %%rax\n\t"
		"jnz	2f\n\t"
		"movq	%[stack], %%rsp\n\t"
		"andq	$-16, %%rsp\n\t"
		"subq	$8, %%rsp\n\t"
		"movq	%[func], %%rdi\n\t"
		"movq	%[data], %%rsi\n\t"
		"xorl	%%ebp, %%ebp\n\t"
		"jmp	__start_thread\n"
		"2:"
		: "+a"(rax),	/* %rax */
		  "+D"(args),	/* %rdi */
		  "+S"(size),	/* %rsi */
		  [func]"+d"(func),
		  [data]"+r"(data),
		  [stack]"+r"(args->stack)
		:
		: "memory", "rcx", "r11"
	);
	return rax;
}

static inline void *malloc(size_t len)
{
	struct simple_heap *h;

	len += sizeof(*h);
	h = __sys_mmap(NULL, len, PROT_READ | PROT_WRITE,
		       MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
	if (IS_ERR(h))
		return NULL;

	return h->user_data;
}

static inline void free(void *p)
{
	struct simple_heap *h;

	h = container_of(p, struct simple_heap, user_data);
	__sys_munmap(h, h->len);
}

static inline uint16_t htons(uint16_t x)
{
	return __builtin_bswap16(x);
}

static void *memset(void *ptr, int c, size_t len)
{
	unsigned char *p = ptr;
	size_t i;

	for (i = 0; i < len; i++)
		p[i] = (unsigned char) c;

	return p;
}

static void *memcpy(void *dst, const void *src, size_t len)
{
	const unsigned char *src_ = src;
	unsigned char *dst_ = dst;
	size_t i;

	for (i = 0; i < len; i++)
		dst_[i] = src_[i];

	return dst_;
}

static size_t strlen(const char *str)
{
	size_t len;

	for (len = 0; str[len]; len++);
	return len;
}

char *strchr(const char *s, int c_in)
{
	const unsigned char *char_ptr;
	const unsigned long int *longword_ptr;
	unsigned long int longword, magic_bits, charmask;
	unsigned char c;
	c = (unsigned char) c_in;

	for (char_ptr = (const unsigned char *) s;
	     ((unsigned long int) char_ptr &(sizeof(longword) - 1)) != 0;
	     ++char_ptr) {
		if (*char_ptr == c)
			return (void*) char_ptr;
		else if (*char_ptr == '\0')
			return NULL;
	}

	longword_ptr = (unsigned long int *) char_ptr;
	magic_bits = -1;
	magic_bits = magic_bits / 0xff * 0xfe << 1 >> 1 | 1;

	charmask = c | (c << 8);
	charmask |= charmask << 16;
	if (sizeof(longword) > 4)
		charmask |= (charmask << 16) << 16;

	for (;;) {
		longword = *longword_ptr++;
		if ((((longword + magic_bits) ^ ~longword) & ~magic_bits) != 0
			||
			((((longword ^ charmask) + magic_bits) ^
				~(longword ^ charmask)) &~magic_bits) != 0) {
			const unsigned char *cp = (const unsigned char *)(longword_ptr - 1);
			if (*cp == c)
				return (char*) cp;
			else if (*cp == '\0')
				return NULL;
			if (*++cp == c)
				return (char*) cp;
			else if (*cp == '\0')
				return NULL;
			if (*++cp == c)
				return (char*) cp;
			else if (*cp == '\0')
				return NULL;
			if (*++cp == c)
				return (char*) cp;
			else if (*cp == '\0')
				return NULL;

			if (sizeof(longword) > 4) {
				if (*++cp == c)
					return (char*) cp;
				else if (*cp == '\0')
					return NULL;
				if (*++cp == c)
					return (char*) cp;
				else if (*cp == '\0')
					return NULL;
				if (*++cp == c)
					return (char*) cp;
				else if (*cp == '\0')
					return NULL;
				if (*++cp == c)
					return (char*) cp;
				else if (*cp == '\0')
					return NULL;
			}
		}
	}

	return NULL;
}

static char *strrchr(const char *s, int c)
{
	const char *found, *p;
	c = (unsigned char) c;

	/* Since strchr is fast, we use it rather than the obvious loop.  */
	if (c == '\0')
		return strchr(s, '\0');

	found = NULL;
	while ((p = strchr(s, c)) != NULL) {
		found = p;
		s = p + 1;
	}
	return (char *) found;
}

static char *basename(const char *filename)
{
	char *p = strrchr(filename, '/');
	return p ? p + 1 : (char *) filename;
}

static char *strstr(char *string, char *substring)
{
	char *a, *b;

	b = substring;
	if (*b == 0)
		return string;

	for ( ; *string != 0; string += 1) {
		if (*string != *b)
			continue;

		a = string;
		while (1) {
			if (*b == 0)
				return string;
			if (*a++ != *b++)
				break;
		}
		b = substring;
	}
	return NULL;
}


static int strncmp(const char *a, const char *b, size_t size)
{
	unsigned int c;
	int diff = 0;

	while (1) {
		if (!size--)
			break;

		diff = (unsigned char)*a++ - (c = (unsigned char)*b++);

		if (diff)
			break;

		if (!c)
			break;
	}

	return diff;
}


/*
 * Ref: https://github.com/torvalds/linux/blob/v6.1/lib/vsprintf.c
 */
#define INT_MAX 0x7fffffff
#define SIGN	1		/* unsigned/signed, must be 1 */
#define LEFT	2		/* left justified */
#define PLUS	4		/* show plus */
#define SPACE	8		/* space if plus */
#define ZEROPAD	16		/* pad with zero, must be 16 == '0' - ' ' */
#define SMALL	32		/* use lowercase in hex (must be 32 == 0x20) */
#define SPECIAL	64		/* prefix hex with "0x", octal with "0" */

static const char hex_asc_upper[] = "0123456789ABCDEF";

enum format_type {
	FORMAT_TYPE_NONE, /* Just a string part */
	FORMAT_TYPE_WIDTH,
	FORMAT_TYPE_PRECISION,
	FORMAT_TYPE_CHAR,
	FORMAT_TYPE_STR,
	FORMAT_TYPE_PTR,
	FORMAT_TYPE_PERCENT_CHAR,
	FORMAT_TYPE_INVALID,
	FORMAT_TYPE_LONG_LONG,
	FORMAT_TYPE_ULONG,
	FORMAT_TYPE_LONG,
	FORMAT_TYPE_UBYTE,
	FORMAT_TYPE_BYTE,
	FORMAT_TYPE_USHORT,
	FORMAT_TYPE_SHORT,
	FORMAT_TYPE_UINT,
	FORMAT_TYPE_INT,
	FORMAT_TYPE_SIZE_T,
	FORMAT_TYPE_PTRDIFF
};

struct printf_spec {
	unsigned int	type:8;		/* format_type enum */
	signed int	field_width:24;	/* width of output field */
	unsigned int	flags:8;	/* flags to number() */
	unsigned int	base:8;		/* number base, 8, 10 or 16 only */
	signed int	precision:16;	/* # of digits/chars */
} __attribute__((__packed__));

#define __ismask(x) (_ctype[(int)(unsigned char)(x)])

#define isalnum(c)	((__ismask(c)&(_U|_L|_D)) != 0)
#define isalpha(c)	((__ismask(c)&(_U|_L)) != 0)
#define iscntrl(c)	((__ismask(c)&(_C)) != 0)
#define isgraph(c)	((__ismask(c)&(_P|_U|_L|_D)) != 0)
#define islower(c)	((__ismask(c)&(_L)) != 0)
#define isprint(c)	((__ismask(c)&(_P|_U|_L|_D|_SP)) != 0)
#define ispunct(c)	((__ismask(c)&(_P)) != 0)
/* Note: isspace() must return false for %NUL-terminator */
#define isspace(c)	((__ismask(c)&(_S)) != 0)
#define isupper(c)	((__ismask(c)&(_U)) != 0)
#define isxdigit(c)	((__ismask(c)&(_D|_X)) != 0)

#define isascii(c) (((unsigned char)(c))<=0x7f)
#define toascii(c) (((unsigned char)(c))&0x7f)

#define tolower(c) __tolower(c)
#define toupper(c) __toupper(c)

/*
 * Fast implementation of tolower() for internal usage. Do not use in your
 * code.
 */
static inline char _tolower(const char c)
{
	return c | 0x20;
}

static int isdigit(int c)
{
	return (unsigned int)(c - '0') < 10;
}

#define __cpu_to_le16(x) (u16)(x)

/**
 * do_div - returns 2 values: calculate remainder and update new dividend
 * @n: uint64_t dividend (will be updated)
 * @base: uint32_t divisor
 *
 * Summary:
 * ``uint32_t remainder = n % base;``
 * ``n = n / base;``
 *
 * Return: (uint32_t)remainder
 *
 * NOTE: macro parameter @n is evaluated multiple times,
 * beware of side effects!
 */
# define do_div(n,base) ({					\
	uint32_t __base = (base);				\
	uint32_t __rem;						\
	__rem = ((uint64_t)(n)) % __base;			\
	(n) = ((uint64_t)(n)) / __base;				\
	__rem;							\
 })


static const u16 decpair[100] = {
#define _(x) (u16) __cpu_to_le16(((x % 10) | ((x / 10) << 8)) + 0x3030)
	_( 0), _( 1), _( 2), _( 3), _( 4), _( 5), _( 6), _( 7), _( 8), _( 9),
	_(10), _(11), _(12), _(13), _(14), _(15), _(16), _(17), _(18), _(19),
	_(20), _(21), _(22), _(23), _(24), _(25), _(26), _(27), _(28), _(29),
	_(30), _(31), _(32), _(33), _(34), _(35), _(36), _(37), _(38), _(39),
	_(40), _(41), _(42), _(43), _(44), _(45), _(46), _(47), _(48), _(49),
	_(50), _(51), _(52), _(53), _(54), _(55), _(56), _(57), _(58), _(59),
	_(60), _(61), _(62), _(63), _(64), _(65), _(66), _(67), _(68), _(69),
	_(70), _(71), _(72), _(73), _(74), _(75), _(76), _(77), _(78), _(79),
	_(80), _(81), _(82), _(83), _(84), _(85), _(86), _(87), _(88), _(89),
	_(90), _(91), _(92), _(93), _(94), _(95), _(96), _(97), _(98), _(99),
#undef _
};

static void *memmove(void *dst, const void *src, size_t len)
{
	size_t dir, pos;

	pos = len;
	dir = -1;

	if (dst < src) {
		pos = -1;
		dir = 1;
	}

	while (len) {
		pos += dir;
		((char *)dst)[pos] = ((const char *)src)[pos];
		len--;
	}
	return dst;
}

static void move_right(char *buf, char *end, unsigned len, unsigned spaces)
{
	size_t size;
	if (buf >= end)	/* nowhere to put anything */
		return;
	size = end - buf;
	if (size <= spaces) {
		memset(buf, ' ', size);
		return;
	}
	if (len) {
		if (len > size - spaces)
			len = size - spaces;
		memmove(buf + spaces, buf, len);
	}
	memset(buf, ' ', spaces);
}


/*
 * This will print a single '0' even if r == 0, since we would
 * immediately jump to out_r where two 0s would be written but only
 * one of them accounted for in buf. This is needed by ip4_string
 * below. All other callers pass a non-zero value of r.
*/
static char *put_dec_trunc8(char *buf, unsigned r)
{
	unsigned q;

	/* 1 <= r < 10^8 */
	if (r < 100)
		goto out_r;

	/* 100 <= r < 10^8 */
	q = (r * (u64)0x28f5c29) >> 32;
	*((u16 *)buf) = decpair[r - 100*q];
	buf += 2;

	/* 1 <= q < 10^6 */
	if (q < 100)
		goto out_q;

	/*  100 <= q < 10^6 */
	r = (q * (u64)0x28f5c29) >> 32;
	*((u16 *)buf) = decpair[q - 100*r];
	buf += 2;

	/* 1 <= r < 10^4 */
	if (r < 100)
		goto out_r;

	/* 100 <= r < 10^4 */
	q = (r * 0x147b) >> 19;
	*((u16 *)buf) = decpair[r - 100*q];
	buf += 2;
out_q:
	/* 1 <= q < 100 */
	r = q;
out_r:
	/* 1 <= r < 100 */
	*((u16 *)buf) = decpair[r];
	buf += r < 10 ? 1 : 2;
	return buf;
}


static char *put_dec_full8(char *buf, unsigned r)
{
	unsigned q;

	/* 0 <= r < 10^8 */
	q = (r * (u64)0x28f5c29) >> 32;
	*((u16 *)buf) = decpair[r - 100*q];
	buf += 2;

	/* 0 <= q < 10^6 */
	r = (q * (u64)0x28f5c29) >> 32;
	*((u16 *)buf) = decpair[q - 100*r];
	buf += 2;

	/* 0 <= r < 10^4 */
	q = (r * 0x147b) >> 19;
	*((u16 *)buf) = decpair[r - 100*q];
	buf += 2;

	/* 0 <= q < 100 */
	*((u16 *)buf) = decpair[q];
	buf += 2;
	return buf;
}

static char *put_dec(char *buf, unsigned long long n)
{
	if (n >= 100*1000*1000)
		buf = put_dec_full8(buf, do_div(n, 100*1000*1000));
	/* 1 <= n <= 1.6e11 */
	if (n >= 100*1000*1000)
		buf = put_dec_full8(buf, do_div(n, 100*1000*1000));
	/* 1 <= n < 1e8 */
	return put_dec_trunc8(buf, n);
}

noinline static char *number(char *buf, char *end, unsigned long long num,
			     struct printf_spec spec)
{
	/* put_dec requires 2-byte alignment of the buffer. */
	char tmp[3 * sizeof(num)] __attribute__((__aligned__(2)));
	char sign;
	char locase;
	int need_pfx = ((spec.flags & SPECIAL) && spec.base != 10);
	int i;
	bool is_zero = num == 0LL;
	int field_width = spec.field_width;
	int precision = spec.precision;

	/* locase = 0 or 0x20. ORing digits or letters with 'locase'
	 * produces same digits or (maybe lowercased) letters */
	locase = (spec.flags & SMALL);
	if (spec.flags & LEFT)
		spec.flags &= ~ZEROPAD;
	sign = 0;
	if (spec.flags & SIGN) {
		if ((signed long long)num < 0) {
			sign = '-';
			num = -(signed long long)num;
			field_width--;
		} else if (spec.flags & PLUS) {
			sign = '+';
			field_width--;
		} else if (spec.flags & SPACE) {
			sign = ' ';
			field_width--;
		}
	}
	if (need_pfx) {
		if (spec.base == 16)
			field_width -= 2;
		else if (!is_zero)
			field_width--;
	}

	/* generate full string in tmp[], in reverse order */
	i = 0;
	if (num < spec.base)
		tmp[i++] = hex_asc_upper[num] | locase;
	else if (spec.base != 10) { /* 8 or 16 */
		int mask = spec.base - 1;
		int shift = 3;

		if (spec.base == 16)
			shift = 4;
		do {
			tmp[i++] = (hex_asc_upper[((unsigned char)num) & mask] | locase);
			num >>= shift;
		} while (num);
	} else { /* base 10 */
		i = put_dec(tmp, num) - tmp;
	}

	/* printing 100 using %2d gives "100", not "00" */
	if (i > precision)
		precision = i;
	/* leading space padding */
	field_width -= precision;
	if (!(spec.flags & (ZEROPAD | LEFT))) {
		while (--field_width >= 0) {
			if (buf < end)
				*buf = ' ';
			++buf;
		}
	}
	/* sign */
	if (sign) {
		if (buf < end)
			*buf = sign;
		++buf;
	}
	/* "0x" / "0" prefix */
	if (need_pfx) {
		if (spec.base == 16 || !is_zero) {
			if (buf < end)
				*buf = '0';
			++buf;
		}
		if (spec.base == 16) {
			if (buf < end)
				*buf = ('X' | locase);
			++buf;
		}
	}
	/* zero or space padding */
	if (!(spec.flags & LEFT)) {
		char c = ' ' + (spec.flags & ZEROPAD);

		while (--field_width >= 0) {
			if (buf < end)
				*buf = c;
			++buf;
		}
	}
	/* hmm even more zero padding? */
	while (i <= --precision) {
		if (buf < end)
			*buf = '0';
		++buf;
	}
	/* actual digits of result */
	while (--i >= 0) {
		if (buf < end)
			*buf = tmp[i];
		++buf;
	}
	/* trailing space padding */
	while (--field_width >= 0) {
		if (buf < end)
			*buf = ' ';
		++buf;
	}

	return buf;
}


noinline static char *widen_string(char *buf, int n, char *end,
				   struct printf_spec spec)
{
	unsigned spaces;

	if (likely(n >= spec.field_width))
		return buf;
	/* we want to pad the sucker */
	spaces = spec.field_width - n;
	if (!(spec.flags & LEFT)) {
		move_right(buf - n, end, n, spaces);
		return buf + spaces;
	}
	while (spaces--) {
		if (buf < end)
			*buf = ' ';
		++buf;
	}
	return buf;
}

static char *string_nocheck(char *buf, char *end, const char *s,
			    struct printf_spec spec)
{
	int len = 0;
	int lim = spec.precision;

	while (lim--) {
		char c = *s++;
		if (!c)
			break;
		if (buf < end)
			*buf = c;
		++buf;
		++len;
	}
	return widen_string(buf, len, end, spec);
}

noinline static char *string(char *buf, char *end, const char *s,
			     struct printf_spec spec)
{
	return string_nocheck(buf, end, s, spec);
}

noinline static int skip_atoi(const char **s)
{
	int i = 0;

	do {
		i = i*10 + *((*s)++) - '0';
	} while (isdigit(**s));

	return i;
}


noinline static int format_decode(const char *fmt, struct printf_spec *spec)
{
	const char *start = fmt;
	char qualifier;

	/* we finished early by reading the field width */
	if (spec->type == FORMAT_TYPE_WIDTH) {
		if (spec->field_width < 0) {
			spec->field_width = -spec->field_width;
			spec->flags |= LEFT;
		}
		spec->type = FORMAT_TYPE_NONE;
		goto precision;
	}

	/* we finished early by reading the precision */
	if (spec->type == FORMAT_TYPE_PRECISION) {
		if (spec->precision < 0)
			spec->precision = 0;

		spec->type = FORMAT_TYPE_NONE;
		goto qualifier;
	}

	/* By default */
	spec->type = FORMAT_TYPE_NONE;

	for (; *fmt ; ++fmt) {
		if (*fmt == '%')
			break;
	}

	/* Return the current non-format string */
	if (fmt != start || !*fmt)
		return fmt - start;

	/* Process flags */
	spec->flags = 0;

	while (1) { /* this also skips first '%' */
		bool found = true;

		++fmt;

		switch (*fmt) {
		case '-': spec->flags |= LEFT;    break;
		case '+': spec->flags |= PLUS;    break;
		case ' ': spec->flags |= SPACE;   break;
		case '#': spec->flags |= SPECIAL; break;
		case '0': spec->flags |= ZEROPAD; break;
		default:  found = false;
		}

		if (!found)
			break;
	}

	/* get field width */
	spec->field_width = -1;

	if (isdigit(*fmt))
		spec->field_width = skip_atoi(&fmt);
	else if (*fmt == '*') {
		/* it's the next argument */
		spec->type = FORMAT_TYPE_WIDTH;
		return ++fmt - start;
	}

precision:
	/* get the precision */
	spec->precision = -1;
	if (*fmt == '.') {
		++fmt;
		if (isdigit(*fmt)) {
			spec->precision = skip_atoi(&fmt);
			if (spec->precision < 0)
				spec->precision = 0;
		} else if (*fmt == '*') {
			/* it's the next argument */
			spec->type = FORMAT_TYPE_PRECISION;
			return ++fmt - start;
		}
	}

qualifier:
	/* get the conversion qualifier */
	qualifier = 0;
	if (*fmt == 'h' || _tolower(*fmt) == 'l' ||
	    *fmt == 'z' || *fmt == 't') {
		qualifier = *fmt++;
		if (unlikely(qualifier == *fmt)) {
			if (qualifier == 'l') {
				qualifier = 'L';
				++fmt;
			} else if (qualifier == 'h') {
				qualifier = 'H';
				++fmt;
			}
		}
	}

	/* default base */
	spec->base = 10;
	switch (*fmt) {
	case 'c':
		spec->type = FORMAT_TYPE_CHAR;
		return ++fmt - start;

	case 's':
		spec->type = FORMAT_TYPE_STR;
		return ++fmt - start;

	case 'p':
		spec->type = FORMAT_TYPE_PTR;
		return ++fmt - start;

	case '%':
		spec->type = FORMAT_TYPE_PERCENT_CHAR;
		return ++fmt - start;

	/* integer number formats - set up the flags and "break" */
	case 'o':
		spec->base = 8;
		break;

	case 'x':
		spec->flags |= SMALL;
		fallthrough;

	case 'X':
		spec->base = 16;
		break;

	case 'd':
	case 'i':
		spec->flags |= SIGN;
		break;
	case 'u':
		break;

	case 'n':
		/*
		 * Since %n poses a greater security risk than
		 * utility, treat it as any other invalid or
		 * unsupported format specifier.
		 */
		fallthrough;

	default:
		spec->type = FORMAT_TYPE_INVALID;
		return fmt - start;
	}

	if (qualifier == 'L')
		spec->type = FORMAT_TYPE_LONG_LONG;
	else if (qualifier == 'l') {
		spec->type = FORMAT_TYPE_ULONG + (spec->flags & SIGN);
	} else if (qualifier == 'z') {
		spec->type = FORMAT_TYPE_SIZE_T;
	} else if (qualifier == 't') {
		spec->type = FORMAT_TYPE_PTRDIFF;
	} else if (qualifier == 'H') {
		spec->type = FORMAT_TYPE_UBYTE + (spec->flags & SIGN);
	} else if (qualifier == 'h') {
		spec->type = FORMAT_TYPE_USHORT + (spec->flags & SIGN);
	} else {
		spec->type = FORMAT_TYPE_UINT + (spec->flags & SIGN);
	}

	return ++fmt - start;
}

static int vsnprintf(char *buf, size_t size, const char *fmt, va_list args)
{
	unsigned long long num;
	char *str, *end;
	struct printf_spec spec = {0};

	str = buf;
	end = buf + size;

		/* Make sure end is always >= buf */
	if (end < buf) {
		end = ((void *)-1);
		size = end - buf;
	}

	while (*fmt) {
		const char *old_fmt = fmt;
		int read = format_decode(fmt, &spec);

		fmt += read;

		switch (spec.type) {
		case FORMAT_TYPE_NONE: {
			int copy = read;
			if (str < end) {
				if (copy > end - str)
					copy = end - str;
				memcpy(str, old_fmt, copy);
			}
			str += read;
			break;
		}

		case FORMAT_TYPE_CHAR: {
			char c;

			if (!(spec.flags & LEFT)) {
				while (--spec.field_width > 0) {
					if (str < end)
						*str = ' ';
					++str;

				}
			}
			c = (unsigned char) va_arg(args, int);
			if (str < end)
				*str = c;
			++str;
			while (--spec.field_width > 0) {
				if (str < end)
					*str = ' ';
				++str;
			}
			break;
		}

		case FORMAT_TYPE_STR:
			str = string(str, end, va_arg(args, char *), spec);
			break;


		case FORMAT_TYPE_PERCENT_CHAR:
			if (str < end)
				*str = '%';
			++str;
			break;

		case FORMAT_TYPE_INVALID:
			/*
			 * Presumably the arguments passed gcc's type
			 * checking, but there is no safe or sane way
			 * for us to continue parsing the format and
			 * fetching from the va_list; the remaining
			 * specifiers and arguments would be out of
			 * sync.
			 */
			goto out;

		default:
			switch (spec.type) {
			case FORMAT_TYPE_LONG_LONG:
				num = va_arg(args, long long);
				break;
			case FORMAT_TYPE_ULONG:
				num = va_arg(args, unsigned long);
				break;
			case FORMAT_TYPE_LONG:
				num = va_arg(args, long);
				break;
			case FORMAT_TYPE_SIZE_T:
				if (spec.flags & SIGN)
					num = va_arg(args, ssize_t);
				else
					num = va_arg(args, size_t);
				break;
			case FORMAT_TYPE_PTRDIFF:
				num = va_arg(args, ptrdiff_t);
				break;
			case FORMAT_TYPE_UBYTE:
				num = (unsigned char) va_arg(args, int);
				break;
			case FORMAT_TYPE_BYTE:
				num = (signed char) va_arg(args, int);
				break;
			case FORMAT_TYPE_USHORT:
				num = (unsigned short) va_arg(args, int);
				break;
			case FORMAT_TYPE_SHORT:
				num = (short) va_arg(args, int);
				break;
			case FORMAT_TYPE_INT:
				num = (int) va_arg(args, int);
				break;
			default:
				num = va_arg(args, unsigned int);
			}

			str = number(str, end, num, spec);
		}
	}

out:
	if (size > 0) {
		if (str < end)
			*str = '\0';
		else
			end[-1] = '\0';
	}

	/* the trailing null byte doesn't count towards the total */
	return str - buf;
}

__maybe_unused
static int vsprintf(char *buf, const char *fmt, va_list args)
{
	return vsnprintf(buf, INT_MAX, fmt, args);
}

/**
 * snprintf - Format a string and place it in a buffer
 * @buf: The buffer to place the result into
 * @size: The size of the buffer, including the trailing null space
 * @fmt: The format string to use
 * @...: Arguments for the format string
 *
 * The return value is the number of characters which would be
 * generated for the given input, excluding the trailing null,
 * as per ISO C99.  If the return is greater than or equal to
 * @size, the resulting string is truncated.
 *
 * See the vsnprintf() documentation for format string extensions over C99.
 */
static int snprintf(char *buf, size_t size, const char *fmt, ...)
{
	va_list args;
	int i;

	va_start(args, fmt);
	i = vsnprintf(buf, size, fmt, args);
	va_end(args);

	return i;
}

static int puts(const char *str)
{
	size_t len = strlen(str);
	int ret;

	if (len <= 4095) {
		char buf[4096];

		memcpy(buf, str, len);
		buf[len] = '\n';
		ret = __sys_write(1, buf, len + 1);
	} else {
		char *buf = malloc(len + 1);

		if (unlikely(!buf))
			return -ENOMEM;

		memcpy(buf, str, len);
		buf[len] = '\n';
		ret = __sys_write(1, buf, len + 1);
		free(buf);
	}
	return ret;
}

static int thread_create(pid_t *pid_p, void *(*target_func)(void *arg),
			 void *arg)
{
	static const size_t stack_size = 1024 * 1024;
	struct clone_args args;
	char *stack;
	pid_t pid;

	stack = __sys_mmap(NULL, stack_size, PROT_READ | PROT_WRITE,
			   MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
	if (IS_ERR(stack))
		return PTR_ERR(stack);

	memset(&args, 0, sizeof(args));
	args.stack = (__aligned_u64)(stack + stack_size - 4096);
	args.stack_size = stack_size - 4096;
	args.flags = CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND |
		     CLONE_THREAD | CLONE_SYSVSEM | CLONE_PARENT_SETTID;

	pid = do_clone(&args, sizeof(args), target_func, arg);
	if (pid < 0) {
		__sys_munmap(stack, stack_size);
		return pid;
	}

	*pid_p = pid;
	return 0;
}

static int run_app_event_loop(struct worker *w);

static void *worker_func(void *arg)
{
	run_app_event_loop(arg);
	return NULL;
}

static int epoll_mod(int epl_fd, int fd, uint32_t events, void *data)
{
	struct epoll_event ev;
	int ret;

	memset(&ev, 0, sizeof(ev));
	ev.events = events;
	ev.data.ptr = data;
	ret = __sys_epoll_ctl(epl_fd, EPOLL_CTL_MOD, fd, &ev);
	if (unlikely(ret)) {
		puts("epoll_ctl(EPOLL_CTL_MOD) error!");
		return ret;
	}

	return 0;
}

static int epoll_add(int epl_fd, int fd, uint32_t events, void *data)
{
	struct epoll_event ev;
	int ret;

	memset(&ev, 0, sizeof(ev));
	ev.events = events;
	ev.data.ptr = data;
	ret = __sys_epoll_ctl(epl_fd, EPOLL_CTL_ADD, fd, &ev);
	if (unlikely(ret)) {
		puts("epoll_ctl(EPOLL_CTL_ADD) error!");
		return ret;
	}

	return 0;
}

static int epoll_delete(int epl_fd, int fd)
{
	int ret;

	ret = __sys_epoll_ctl(epl_fd, EPOLL_CTL_DEL, fd, NULL);
	if (unlikely(ret)) {
		puts("epoll_ctl(EPOLL_CTL_DEL) error!");
		return ret;
	}

	return 0;
}

static int init_server_socket(struct server_ctx *sctx)
{
	struct sockaddr_in saddr;
	int ret;
	int fd;

	fd = __sys_socket(AF_INET, SOCK_STREAM | SOCK_NONBLOCK, 0);
	if (unlikely(fd < 0)) {
		puts("socket() error!");
		return fd;
	}

	ret = 1;
	__sys_setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &ret, sizeof(ret));
	__sys_setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &ret, sizeof(ret));

	memset(&saddr, 0, sizeof(saddr));
	saddr.sin_family = AF_INET;
	saddr.sin_port = htons(BIND_PORT);

	ret = __sys_bind(fd, (void *)&saddr, sizeof(saddr));
	if (unlikely(ret < 0)) {
		puts("bind() error!");
		__sys_close(fd);
		return ret;
	}

	ret = __sys_listen(fd, 1024);
	if (unlikely(ret < 0)) {
		puts("listen() error!");
		__sys_close(fd);
		return ret;
	}

	puts("Listening 0.0.0.0:9999...");
	sctx->tcp_fd = fd;
	return 0;
}

static int init_server_worker(struct worker *w)
{
	int epl_fd;
	int ret;

	epl_fd = __sys_epoll_create(100);
	if (epl_fd < 0) {
		puts("epoll_create() error!");
		return epl_fd;
	}
	w->epl_fd = epl_fd;

	if (w->idx == 0) {
		w->pid = 0;
		ret = epoll_add(epl_fd, w->sctx->tcp_fd, EPOLLIN | EPOLLPRI,
				NULL);
		if (unlikely(ret))
			goto err;
	} else {
		ret = thread_create(&w->pid, worker_func, w);
		if (unlikely(ret < 0))
			goto err;
	}

	return 0;

err:
	w->epl_fd = -1;
	__sys_close(epl_fd);
	return ret;
}

static int init_server_workers(struct server_ctx *sctx)
{
	struct worker *workers;
	size_t nr_workers = 1;
	size_t i;
	int ret;

	while (nr_workers < NR_WORKERS)
		nr_workers *= 2;

	workers = malloc(sizeof(*workers) * nr_workers);
	if (unlikely(!workers))
		return -ENOMEM;

	sctx->pos_worker = 0;
	sctx->workers = workers;
	sctx->nr_workers = nr_workers;

	for (i = 0; i < nr_workers; i++) {
		struct worker *w = &workers[i];

		w->idx = i;
		w->sctx = sctx;
		ret = init_server_worker(w);
		if (unlikely(ret)) {
			free(workers);
			return ret;
		}
	}

	mutex_init(&sctx->worker_mutex);
	return 0;
}

static int pop_stack(struct stack *stack, uint32_t *val)
{
	int ret;

	mutex_lock(&stack->mutex);
	if (unlikely(stack->sp == stack->max)) {
		ret = -EAGAIN;
	} else {
		ret = 0;
		*val = stack->arr[stack->sp++];
	}
	mutex_unlock(&stack->mutex);
	return ret;
}

static int push_stack(struct stack *stack, uint32_t val)
{
	int ret;

	mutex_lock(&stack->mutex);
	if (unlikely(!stack->sp)) {
		ret = -EAGAIN;
	} else {
		ret = 0;
		stack->arr[--stack->sp] = val;
	}
	mutex_unlock(&stack->mutex);
	return ret;
}

static int init_client_slots(struct server_ctx *sctx)
{
	struct stack *stack = &sctx->client_stk;
	u32 nr_clients = NR_CLIENTS;
	u32 *arr;

	arr = malloc(NR_CLIENTS * sizeof(*arr));
	if (unlikely(!arr))
		return -ENOMEM;

	stack->arr = arr;
	stack->max = NR_CLIENTS;
	stack->sp = NR_CLIENTS;
	mutex_init(&stack->mutex);

	while (nr_clients--) {
		sctx->clients[nr_clients].idx = nr_clients;
		stack->arr[--stack->sp] = nr_clients;
	}

	return 0;
}

static int init_server_ctx(struct server_ctx *sctx)
{
	int ret;

	ret = init_server_socket(sctx);
	if (unlikely(ret))
		return ret;

	ret = init_client_slots(sctx);
	if (unlikely(ret)) {
		__sys_close(sctx->tcp_fd);
		return ret;
	}

	ret = init_server_workers(sctx);
	if (unlikely(ret)) {
		__sys_close(sctx->tcp_fd);
		free(sctx->client_stk.arr);
		return ret;
	}

	return 0;
}

static struct client *get_client_slot(struct server_ctx *sctx)
{
	struct client *cl;
	u32 idx;

	if (unlikely(pop_stack(&sctx->client_stk, &idx)))
		return NULL;

	cl = &sctx->clients[idx];
	cl->buf_len = 0;
	cl->pfile.map = NULL;
	cl->pfile.pos = 0;
	cl->pfile.len = 0;
	return cl;
}

static void put_client_slot(struct worker *w, struct server_ctx *sctx,
			    struct client *cl)
{
	int fd;

	fd = cl->fd;
	epoll_delete(w->epl_fd, fd);
	__sys_close(fd);

	if (cl->pfile.map)
		__sys_munmap(cl->pfile.map, cl->pfile.len);

	if (push_stack(&sctx->client_stk, cl->idx)) {
		puts("Malformed put_client_slot!");
		__sys_exit_group(-1);
	}
}

static int register_client_to_worker(struct server_ctx *sctx, struct client *cl)
{
	struct worker *workers = sctx->workers;
	struct worker *to;
	u32 mask;
	u32 idx;

	mask = sctx->nr_workers - 1;

	mutex_lock(&sctx->worker_mutex);
	idx = sctx->pos_worker++ & mask;
	to = &workers[idx];
	mutex_unlock(&sctx->worker_mutex);

	return epoll_add(to->epl_fd, cl->fd, EPOLLIN | EPOLLPRI, cl);
}

static int handle_new_connection(struct worker *w, struct server_ctx *sctx,
				 int tcp_fd)
{
	struct client *cl;
	struct sockaddr_in addr4;
	socklen_t addrlen;
	int fd;

	if (unlikely(w->acc_cycle++ >= 32))
		return -EAGAIN;

	addrlen = sizeof(addr4);
	fd = __sys_accept4(tcp_fd, (void *)&addr4, &addrlen, SOCK_NONBLOCK);
	if (fd < 0) {
		if (unlikely(fd != -EAGAIN))
			puts("accept() error!");

		return fd;
	}

	cl = get_client_slot(sctx);
	if (unlikely(!cl)) {
		puts("Client slot is full, can't accept more connection...");
		__sys_close(fd);
		return -EAGAIN;
	}

	cl->addr4 = addr4;
	cl->fd = fd;
	return register_client_to_worker(sctx, cl);
}

static int handle_new_connections(struct worker *w, struct server_ctx *sctx,
				  struct epoll_event *event)
{
	int tcp_fd;
	int ret;

	if (unlikely(event->events & (EPOLLERR | EPOLLHUP))) {
		puts("The main TCP socket is down!");
		return -ENETDOWN;
	}

	tcp_fd = sctx->tcp_fd;
	w->acc_cycle = 0;
	while (1) {
		ret = handle_new_connection(w, sctx, tcp_fd);

		if (likely(ret == -EAGAIN))
			return 0;

		if (unlikely(ret < 0))
			return ret;
	}
}

static int send_request_not_supported(struct worker *w, struct server_ctx *sctx,
				      struct client *cl)
{
	static const char msg[] =
		"HTTP/1.1 400 Bad Request\r\n"
		"Content-Type: text/plain\r\n"
		"Content-Length: 56\r\n\r\n"
		"Sorry, this type of request is currently not supported!\n";

	__sys_write(cl->fd, msg, sizeof(msg) - 1);
	put_client_slot(w, sctx, cl);
	return 0;
}

static int send_bad_request(struct worker *w, struct server_ctx *sctx,
			    struct client *cl)
{
	static const char msg[] =
		"HTTP/1.1 400 Bad Request\r\n"
		"Content-Type: text/plain\r\n"
		"Content-Length: 13\r\n\r\n"
		"Bad request!\n";

	__sys_write(cl->fd, msg, sizeof(msg) - 1);
	put_client_slot(w, sctx, cl);
	return 0;
}

static int send_not_found(struct worker *w, struct server_ctx *sctx,
			  struct client *cl)
{
	static const char msg[] =
		"HTTP/1.1 404 Not Found\r\n"
		"Content-Type: text/plain\r\n"
		"Content-Length: 11\r\n\r\n"
		"Not Found!\n";

	__sys_write(cl->fd, msg, sizeof(msg) - 1);
	put_client_slot(w, sctx, cl);
	return 0;
}


static int send_redirect(struct worker *w, struct server_ctx *sctx,
			 struct client *cl, const char *target)
{
	static const char msg[] =
		"HTTP/1.1 301 Moved Permanently\r\n"
		"Location: %s\r\n"
		"Content-Type: text/plain\r\n"
		"Content-Length: 12\r\n\r\n"
		"Redirected!\n";

	char tmp[sizeof(msg) + 512];
	int ret;

	ret = snprintf(tmp, sizeof(tmp), msg, target);
	__sys_write(cl->fd, tmp, ret);
	put_client_slot(w, sctx, cl);
	return 0;
}

static int parse_uri(struct client *cl)
{
	const size_t shift = sizeof("GET ") - 1;
	char *buf = cl->buf + shift;
	char *p = buf;
	size_t i = shift;

	while (1) {
		if (i++ >= cl->buf_len)
			return -EINVAL;

		if (i >= (shift + 255))
			return -EINVAL;

		if (*p == ' ') {
			p[0] = '\0';
			break;
		}

		p++;
	}

	cl->uri = buf;
	return 0;
}

static void __build_directory_list(const char *path, char **p_buf,
				   size_t *p_len, size_t *p_capacity,
				   struct linux_dirent64 *d)
{
	char fname[512];
	const char *type;
	const char *f;
	struct stat st;
	char size_str[64];
	int ret;

	f = d->d_name;
	if (unlikely(!strncmp(f, ".", 2) || !strncmp(f, "..", 3)))
		return;

	snprintf(fname, sizeof(fname), "%s/%s", path, f);
	if (unlikely(__sys_stat(fname, &st)))
		return;

	size_str[0] = '\0';
	if (S_ISDIR(st.st_mode))
		type = "Directory";
	else if (S_ISREG(st.st_mode)) {
		type = "Regular File";
		snprintf(size_str, sizeof(size_str), "%llu",
			 (unsigned long long)st.st_size);
	} else if (S_ISCHR(st.st_mode))
		type = "Char dev";
	else if (S_ISBLK(st.st_mode))
		type = "Block dev";
	else if (S_ISFIFO(st.st_mode))
		type = "FIFO";
	else if (S_ISLNK(st.st_mode))
		type = "Symlink";
	else if (S_ISSOCK(st.st_mode))
		type = "Socket";
	else
		return;

	ret = snprintf(*p_buf, *p_capacity,
		"\t\t<tr>"
			"<td><a href=\"%s%s\">%s</a></td>"
			"<td>%s</td>"
			"<td>%d%d%d%d</td>"
			"<td>%s</td>"
		"</tr>\n",
		f, S_ISDIR(st.st_mode) ? "/" : "", f,
		type,
		(st.st_mode & 07000) >> 9,
		(st.st_mode & 00700) >> 6,
		(st.st_mode & 00070) >> 3,
		(st.st_mode & 00007),
		size_str
	);
	*p_buf += (size_t)ret;
	*p_len += (size_t)ret;
	*p_capacity -= (size_t)(ret - 1);
}

static int _build_directory_list(const char *path, char *dbuf, ssize_t nread,
				 char **p_buf, size_t *p_len,
				 size_t *p_capacity)
{
	struct linux_dirent64 *d;
	long pos;

	for (pos = 0; pos < nread;) {
		d = (struct linux_dirent64 *)(dbuf + pos);
		__build_directory_list(path, p_buf, p_len, p_capacity, d);
		pos += d->d_reclen;
	}

	return 0;
}

static int build_directory_list(const char *path, int dir_fd, char *buf,
				size_t *p_len, size_t capacity)
{
	char dbuf[1024];
	size_t len = 0;
	ssize_t nread;

	dbuf[sizeof(dbuf) - 1] = '\0';
	for (;;) {
		size_t tmp_len = 0;
		int ret;

		nread = __sys_getdents64(dir_fd, dbuf, sizeof(dbuf) - 1);
		if (unlikely(nread < 0))
			return (int)nread;

		if (!nread)
			break;

		ret = _build_directory_list(path, dbuf, nread, &buf, &tmp_len,
					    &capacity);
		len += tmp_len;
		if (ret)
			break;
	}

	*p_len = len;
	return 0;
}

static int build_directory_list_buffer(const char *path, int dir_fd, char *buf,
				       size_t *p_len, size_t capacity)
{
	static const char hdr[] =
		"HTTP/1.1 200\r\n"
		"Content-Type: text/html\r\n"
		"Connection: closed\r\n\r\n"
		"<!DOCTYPE html>\n"
		"<html>\n"
		"<style type=\"text/css\">"
			"td {padding: 10px;}"
			"a {color: blue; text-decoration: none}"
			"a:hover {text-decoration: underline}"
		"</style>"
		"<body>\n"
		"\t<h1>Extreme GNU/Weeb HTTP Server</h1>\n"
		"\t<table border=\"1\">\n"
		"\t\t<tr>"
			"<th>Filename</th>"
			"<th>Type</th>"
			"<th>Mode</th>"
			"<th>Size</th>"
		"</tr>\n";

	static const char foot[] =
		"\t</table>\n</body>\n</html>\n";

	size_t len;
	int ret;

	capacity -= sizeof(hdr) - sizeof(foot);
	memcpy(buf, hdr, sizeof(hdr) - 1);
	*p_len = sizeof(hdr) - 1;

	ret = build_directory_list(path, dir_fd, &buf[*p_len], &len, capacity);
	if (unlikely(ret))
		return ret;

	*p_len += len;
	memcpy(&buf[*p_len], foot, sizeof(foot) - 1);
	*p_len += sizeof(foot) - 1;
	return 0;
}

static int show_directory_list(struct worker *w, struct server_ctx *sctx,
			       struct client *cl, const char *path, int dir_fd)
{
	char buf[16 * 4096];
	size_t len;
	int ret;

	if (dir_fd == -1) {
		dir_fd = __sys_open(path, O_RDONLY, 0);
		if (unlikely(dir_fd < 0))
			return send_not_found(w, sctx, cl);
	}

	ret = build_directory_list_buffer(path, dir_fd, buf, &len, sizeof(buf));
	__sys_close(dir_fd);
	if (unlikely(ret))
		goto out;

	__sys_write(cl->fd, buf, len);
out:
	put_client_slot(w, sctx, cl);
	return 0;
}

static int handle_directory_path(struct worker *w, struct server_ctx *sctx,
				 struct client *cl, int dir_fd)
{
	if (unlikely(cl->uri[strlen(cl->uri) - 1] != '/')) {
		char uri[512];

		snprintf(uri, sizeof(uri), "%s/", cl->uri);
		return send_redirect(w, sctx, cl, uri);
	}

	return show_directory_list(w, sctx, cl, &cl->uri[1], dir_fd);
}

static int send_headers_for_regular_file(u64 size, struct client *cl)
{
	char buf[1024];
	int wr_size;
	int ret;

	wr_size = snprintf(buf, sizeof(buf),
		"HTTP/1.1 200 OK\r\n"
		"Content-Length: %llu\r\n"
		"Content-Type: application/octet-stream\r\n"
		"Content-Disposition: attachment; filename=\"%s\"\r\n\r\n",
		(unsigned long long)size,
		basename(&cl->uri[1]));

	ret = __sys_write(cl->fd, buf, wr_size);
	if (unlikely(ret != wr_size))
		return -1;

	return 0;
}

static int continue_handle_regular_file_path(struct worker *w,
					     struct server_ctx *sctx,
					     struct client *cl)
{
	size_t wr_size;
	ssize_t ret;
	size_t pos;
	char *map;

	wr_size = cl->pfile.len - cl->pfile.pos;
	if (wr_size >= INT_MAX)
		wr_size = INT_MAX - 1;

	map = cl->pfile.map;
	pos = cl->pfile.pos;
	ret = __sys_write(cl->fd, &map[pos], wr_size);
	if (unlikely(ret < 0)) {
		if (ret != -EAGAIN)
			goto out_put;

		return 0;
	}

	pos += (size_t)ret;
	if (pos >= cl->pfile.len)
		goto out_put;

	cl->pfile.pos = pos;
	return 0;

out_put:
	put_client_slot(w, sctx, cl);
	return 0;
}

static int handle_regular_file_path(struct worker *w, struct server_ctx *sctx,
				    struct client *cl, struct stat *st, int fd)
{
	size_t wr_size;
	ssize_t ret;
	char *map;

	map = __sys_mmap(NULL, st->st_size, PROT_READ, MAP_SHARED, fd, 0);
	__sys_close(fd);
	if (IS_ERR(map))
		return send_not_found(w, sctx, cl);

	wr_size = st->st_size;
	if (wr_size >= INT_MAX)
		wr_size = INT_MAX - 1;

	wr_size = 1;

	ret = send_headers_for_regular_file(st->st_size, cl);
	if (unlikely(ret < 0))
		goto out_unmap_and_put;

	ret = __sys_write(cl->fd, map, wr_size);
	if (unlikely(ret < 0)) {
		if (ret == -EAGAIN) {
			/*
			 * The write buffer is full. Queue it.
			 */
			ret = 0;
			goto out_queue;
		}

		/*
		 * __sys_write() error!
		 */
		goto out_unmap_and_put;
	}

	/*
	 * We finished in a single write.
	 */
	if (ret == st->st_size)
		goto out_unmap_and_put;

out_queue:
	if (epoll_mod(w->epl_fd, cl->fd, EPOLLOUT, cl))
		goto out_unmap_and_put;

	cl->pfile.map = map;
	cl->pfile.pos = (size_t)ret;
	cl->pfile.len = st->st_size;
	return 0;

out_unmap_and_put:
	__sys_munmap(map, st->st_size);
	put_client_slot(w, sctx, cl);
	return 0;
}

static int _do_routing(struct worker *w, struct server_ctx *sctx,
		       struct client *cl)
{
	struct stat st;
	int fd;

	fd = __sys_open(&cl->uri[1], O_RDONLY, 0);
	if (unlikely(fd < 0))
		return send_not_found(w, sctx, cl);

	if (unlikely(__sys_fstat(fd, &st))) {
		__sys_close(fd);
		return send_not_found(w, sctx, cl);
	}

	if (S_ISDIR(st.st_mode))
		return handle_directory_path(w, sctx, cl, fd);

	if (S_ISREG(st.st_mode))
		return handle_regular_file_path(w, sctx, cl, &st, fd);

	__sys_close(fd);
	return send_request_not_supported(w, sctx, cl);
}

static int do_routing(struct worker *w, struct server_ctx *sctx,
		      struct client *cl)
{
	if (cl->uri[0] == '/' && cl->uri[1] == '\0')
		return show_directory_list(w, sctx, cl, ".", -1);

	return _do_routing(w, sctx, cl);
}

static int __handle_client_data(struct worker *w, struct server_ctx *sctx,
				struct client *cl)
{
	char *buf = cl->buf;

	if (strncmp(buf, STRL("GET /")))
		return send_request_not_supported(w, sctx, cl);

	if (parse_uri(cl))
		return send_bad_request(w, sctx, cl);

	return do_routing(w, sctx, cl);
}

static int _handle_client_data(struct worker *w, struct server_ctx *sctx,
			       struct client *cl)
{
	ssize_t len;

	len = __sys_read(cl->fd, cl->buf, sizeof(cl->buf));
	if (unlikely(len < 0)) {

		if (len == -EAGAIN)
			return 0;

		puts("read error!");
		goto out_close;
	}

	if (!len)
		goto out_close;

	if (!strstr(cl->buf, "\r\n\r\n"))
		goto out_close;

	cl->buf_len = (size_t)len;
	return __handle_client_data(w, sctx, cl);

out_close:
	put_client_slot(w, sctx, cl);
	return 0;
}

static int handle_client_data(struct worker *w, struct server_ctx *sctx,
			      struct epoll_event *event)
{
	struct client *cl = event->data.ptr;

	if (unlikely(event->events & (EPOLLERR | EPOLLHUP))) {
		put_client_slot(w, sctx, cl);
		return 0;
	}

	if (likely(event->events & EPOLLIN))
		return _handle_client_data(w, sctx, cl);

	if (likely(event->events & EPOLLOUT))
		return continue_handle_regular_file_path(w, sctx, cl);

	puts("Invalid events in handle_client_data");
	return -EINVAL;
}

static int handle_event(struct worker *w, struct server_ctx *sctx,
			struct epoll_event *event)
{
	if (event->data.u64 == 0)
		return handle_new_connections(w, sctx, event);
	else
		return handle_client_data(w, sctx, event);
}

static int handle_events(struct worker *w, struct server_ctx *sctx, int n)
{
	struct epoll_event *events = w->events;
	int ret;
	int i;

	for (i = 0; i < n; i++) {
		ret = handle_event(w, sctx, &events[i]);
		if (unlikely(ret))
			return ret;
	}

	return 0;
}

static int _run_app_event_loop(struct worker *w, struct server_ctx *sctx)
{
	int ret;

	ret = __sys_epoll_wait(w->epl_fd, w->events, NR_MAX_EVENTS, -1);
	if (unlikely(ret < 0)) {

		if (ret == -EINTR)
			return 0;

		puts("epoll_wait() error!");
		return ret;
	}

	return handle_events(w, sctx, ret);
}

noinline static int run_app_event_loop(struct worker *w)
{
	struct server_ctx *sctx = w->sctx;
	int ret;

	while (1) {
		ret = _run_app_event_loop(w, sctx);
		if (unlikely(ret))
			break;
	}

	puts("run_app_event_loop stopped!");
	return ret;
}

static int setup_signal_handler(void)
{
	struct sigaction act;

	memset(&act, 0, sizeof(act));
	act._u._sa_handler = SIG_IGN;
	return __sys_rt_sigaction(SIGPIPE, &act, NULL, sizeof(sigset_t));
}

static int _main(void)
{
	struct server_ctx *sctx;
	int ret;

	ret = setup_signal_handler();
	if (ret)
		return -ret;

	sctx = malloc(sizeof(*sctx));
	if (!sctx)
		return ENOMEM;

	memset(sctx, 0, sizeof(*sctx));
	ret = init_server_ctx(sctx);
	if (ret < 0)
		return -ret;

	ret = run_app_event_loop(&sctx->workers[0]);
	if (ret < 0)
		return -ret;

	return 0;
}

__used noreturn static void __start(void)
{
	__sys_exit_group(_main());
}