simonhf · September 9, 2019 04:28
diff --git a/_golang_tftpd.md b/_golang_tftpd.md
diff --git a/shftftpd-main.c b/shftftpd-main.c
 #include "shftftpd.c"

 int main(int argc, char * argv)
 {
    tftp_init();

    while(1) {
        if (socket_id_list_ready.total > 0) {
            while (socket_id_list_ready.total > 0) {
                int socket_id = socket_id_list_pop(&socket_id_list_ready);
                printf("%f %6lu=tid socket_id #%d of %u has transferred a %'u byte file '%.*s'\n", get_time_in_seconds(), gettid(), 1 + socket_id, EPOLL_LISTENING_PORTS, epoll_user_data[socket_id].mmap_buffer.bytes_used, epoll_user_data[socket_id].file_name_len, &epoll_user_data[socket_id].file_name[0]);
                //fixme: do something with transferred file bytes
                epoll_user_data[socket_id].mmap_buffer.bytes_used = 0; // truncate mmap buffer for this socket_id
                socket_id_list_push(&socket_id_list_unused, socket_id); // allow socket_id to be used by next tftp client
            }
            socket_id_list_ready.total = 0;
        }
        else {
            sleep_ms(2); // poll for ready tftp files
        }
    }

    return EXIT_SUCCESS;
 } // main()
diff --git a/shftftpd.c b/shftftpd.c
 #define _GNU_SOURCE
 #include <sys/mman.h>       // for mmap() and mremap()
 #include <stdio.h>          // for printf() and fprintf()
 #include <sys/socket.h>     // for socket(), bind(), and connect()
 #include <arpa/inet.h>      // for sockaddr_in and inet_ntoa()
 #include <stdlib.h>         // for atoi() and exit()
 #include <string.h>         // for memset()
 #include <unistd.h>         // for close()
 #include <fcntl.h>          // for fcntl()
 #include <sys/time.h>       // for gettimeofday()
 #include <ctype.h>          // for tolower()
 #include <locale.h>         // for setlocale()
 #include <pthread.h>        // for pthread_self()
 #include <errno.h>
 #include <sys/epoll.h>
 #include <unistd.h>
 #include <sys/syscall.h>

 #ifdef SYS_gettid
 #define gettid() (syscall(SYS_gettid))
 #else
 #error "SYS_gettid unavailable on this system"
 #endif

 #if 1==SHF_DEBUG_VERSION
 #define SHF_DEBUG(ARGS...) printf(ARGS);
 #else
 #define SHF_DEBUG(ARGS...)
 #endif

 #define SHF_CAST(TYPE, PTR) ((TYPE)(uintptr_t)(PTR))
 #define SHF_U08_AT(BASE, OFFSET)                        SHF_CAST(uint8_t *, BASE)[OFFSET]
 #define SHF_U16_AT(BASE, OFFSET) SHF_CAST(uint16_t *, &(SHF_CAST(uint8_t *, BASE)[OFFSET]))[0]

 #define SHF_ASSERT(         CONDITION,FORMAT,ARGS...) if (!(CONDITION)) { printf("%f %6lu=tid %05u:%s: ERROR: assertion: %s in function %s() "                 "// ", get_time_in_seconds(), gettid(), __LINE__, __FILE__, #CONDITION, __FUNCTION__                 ); printf(FORMAT, ARGS); exit(EXIT_FAILURE); }
 #define SHF_ASSERT_STRERROR(CONDITION,FORMAT,ARGS...) if (!(CONDITION)) { printf("%f %6lu=tid %05u:%s: ERROR: assertion: %s in function %s() with strerror() %s // ", get_time_in_seconds(), gettid(), __LINE__, __FILE__, #CONDITION, __FUNCTION__, strerror(errno)); printf(FORMAT, ARGS); exit(EXIT_FAILURE); }

 void sleep_ms(int milliseconds)
 {
 #if _POSIX_C_SOURCE >= 199309L
    struct timespec ts;
    ts.tv_sec = milliseconds / 1000;
    ts.tv_nsec = (milliseconds % 1000) * 1000000;
    nanosleep(&ts, NULL);
 #else
    usleep(milliseconds * 1000);
 #endif
 }

 double get_time_in_seconds(void)
 {
    struct timeval tv;

    int result = gettimeofday(&tv, NULL); SHF_ASSERT_STRERROR(result >= 0, "%d=result=gettimeofday()\n", result);
    return (double)tv.tv_sec + 1.e-6 * (double)tv.tv_usec;
 }

 // hex_dump("buf", buf, buf_length);
 // - 0000  68 65 6c 6c 6f 20 77 6f  72 6c 64 20 61 6e 64 20  |hello world and | <- buf[30]
 // - 0010  67 6f 6f 64 62 79 65 20  77 6f 72 6c 64 0a        |goodbye world.|

 void hex_dump (const char * desc, const void * addr, const int len)
 {
 #if 1==SHF_DEBUG_VERSION
    unsigned char ascii[17];
    const unsigned char *pc = (const unsigned char *)addr;

    SHF_ASSERT(0 != len, "%d=len; len should not be 0\n", len);
    SHF_ASSERT(0 <  len, "%d=len; len should be > 0\n"  , len);

    int i;
    for(i = 0; i < len; i++) {
        if (i > 0 && (i % 8) == 0) { SHF_DEBUG(" "); }

        if ((i % 16) == 0) {
            if (i != 0) {
                SHF_DEBUG(" |%s|", ascii);
                if (i <= 16) { SHF_DEBUG(" <- %s[%u]", desc, len); }
                SHF_DEBUG("\n");
            }
            SHF_DEBUG("%f %6lu=tid hex dump %04x ", get_time_in_seconds(), gettid(), i); //fixme: modify function to only printf() complete lines; better for multi-threaded operation
        }

        SHF_DEBUG(" %02x", pc[i]);

        if ((pc[i] < 0x20) || (pc[i] > 0x7e)) { ascii[i % 16] = '.'  ; }
        else                                  { ascii[i % 16] = pc[i]; }
        ascii[(i % 16) + 1] = '\0';
    }

    while ((i % 16) != 0) {
        if ((i % 8) == 0) { SHF_DEBUG(" "); }
        SHF_DEBUG("   ");
        i ++;
    }
    SHF_DEBUG("  |%s|\n", ascii);
 #endif
 }

 #define UDP_BUF_SIZE (65536)
 #define PAGE_SIZE (4096) //fixme: find the page size programmatically
 #define EPOLL_EVENTS_MAX (100) //fixme: allow to be specified on the command line for performance testing
 #define EPOLL_LISTENING_PORTS (4) //fixme: allow to be specified on command line and check if enough process fds available
 #define EPOLL_LISTENING_PORT_TFTP (6969) //fixme: allow tftp main port to be specified on the command line
 #define TFTP_FILE_NAME_LEN_MAX (512) //fixme: figure out from rfc1350 what the actual max length is!

 typedef enum TFTP_PACKET_TYPE { // see https://tools.ietf.org/html/rfc1350
    TFTP_PACKET_TYPE_RRQ = 1,
    TFTP_PACKET_TYPE_WRQ    ,
    TFTP_PACKET_TYPE_DATA   ,
    TFTP_PACKET_TYPE_ACK    ,
    TFTP_PACKET_TYPE_ERROR  ,
 } TFTP_PACKET_TYPE;

 typedef enum SOCKET_ID_STATUS {
    SOCKET_ID_STATUS_UNUSED = 0,
    SOCKET_ID_STATUS_WAITING_FOR_PEER_WRITE,
 } SOCKET_ID_STATUS;

 typedef struct MMAP_BUFFER {
    void     * bytes;
    uint32_t   bytes_size;
    uint32_t   bytes_used;
 } __attribute__((packed)) MMAP_BUFFER;

 typedef struct EPOLL_USER_DATA {
    SOCKET_ID_STATUS   socket_status;
    int                fd;
    uint16_t           hostshort;
    struct sockaddr_in udp_peer_address;
    int                udp_peer_address_len;
    int                udp_peer_port;
    char               udp_peer_ipv4[INET_ADDRSTRLEN];
    MMAP_BUFFER        mmap_buffer; //fixme: consider backing file to /dev/shm so that /dev/shm becomes the in-memory 'hash table'?
    char               file_name[TFTP_FILE_NAME_LEN_MAX];
    int                file_name_len;
    int                tftp_block;
    double             timestamp_1st;
    double             timestamp_ttl; //fixme: add code to timeout if next packet takes too long; consider chronological double linked list
 } __attribute__((packed)) EPOLL_USER_DATA;

 typedef struct SOCKET_ID_LIST {
    pthread_mutex_t   lock;
    volatile uint16_t list[EPOLL_LISTENING_PORTS];
    volatile int      total;
 } __attribute__((packed)) SOCKET_ID_LIST;

 EPOLL_USER_DATA epoll_user_data[EPOLL_LISTENING_PORTS];
 unsigned char   udp_buf[UDP_BUF_SIZE];
 SOCKET_ID_LIST  socket_id_list_unused;
 SOCKET_ID_LIST  socket_id_list_ready;

 void socket_id_list_init(SOCKET_ID_LIST * socket_id_list)
 {
    int result = pthread_mutex_init(&socket_id_list->lock, NULL); SHF_ASSERT_STRERROR(0 == result, "%d=result=pthread_mutex_init() should be 0\n", result);
    socket_id_list->total = 0;
 }

 void socket_id_list_push(SOCKET_ID_LIST * socket_id_list, int socket_id)
 {
    SHF_ASSERT(socket_id             < EPOLL_LISTENING_PORTS, "%d=socket_id"           "should be < %u; internal error?\n", socket_id            , EPOLL_LISTENING_PORTS);
    SHF_ASSERT(socket_id_list->total < EPOLL_LISTENING_PORTS, "%d=socket_id_list->total should be < %u; internal error?\n", socket_id_list->total, EPOLL_LISTENING_PORTS);
    pthread_mutex_lock(&socket_id_list->lock);
    socket_id_list->list[socket_id_list->total] = socket_id; //fixme: consider randomizing how the socket_id gets added to the list?
    socket_id_list->total ++;
    pthread_mutex_unlock(&socket_id_list->lock);
 }

 int socket_id_list_pop(SOCKET_ID_LIST * socket_id_list)
 {
    int socket_id = -1; // means no free socket_id available
    pthread_mutex_lock(&socket_id_list->lock);
    if (socket_id_list->total > 0) {
        socket_id_list->total --;
        socket_id = socket_id_list->list[socket_id_list->total];
    }
    pthread_mutex_unlock(&socket_id_list->lock);
    return socket_id;
 }

 void mmap_buffer_append(int socket_id, char * src_bytes, uint32_t src_len)
 {
    int mmap_grow_size = 256 * PAGE_SIZE;
    if (0 == epoll_user_data[socket_id].mmap_buffer.bytes_size) {
        // come here if mmap_buffer for this socket_id never created
        while(epoll_user_data[socket_id].mmap_buffer.bytes_size < src_len) { epoll_user_data[socket_id].mmap_buffer.bytes_size += mmap_grow_size; }
        SHF_DEBUG("%f %6lu=tid socket_id #%d of %u mmap_buffer[+%u = %u bytes] for appending +%u = %u bytes\n", get_time_in_seconds(), gettid(), 1 + socket_id, EPOLL_LISTENING_PORTS, epoll_user_data[socket_id].mmap_buffer.bytes_size, epoll_user_data[socket_id].mmap_buffer.bytes_size, src_len, src_len);
        epoll_user_data[socket_id].mmap_buffer.bytes = mmap(NULL, epoll_user_data[socket_id].mmap_buffer.bytes_size, PROT_READ|PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0); SHF_ASSERT_STRERROR(MAP_FAILED != epoll_user_data[socket_id].mmap_buffer.bytes, "%p=mmap()", epoll_user_data[socket_id].mmap_buffer.bytes);
    }
    else if ((src_len + epoll_user_data[socket_id].mmap_buffer.bytes_used) > epoll_user_data[socket_id].mmap_buffer.bytes_size) {
        // come here if mmap_buffer for this socket_id already created, but is it big enough?
        uint32_t old_byte_size = epoll_user_data[socket_id].mmap_buffer.bytes_size;
        while((src_len + epoll_user_data[socket_id].mmap_buffer.bytes_used) > epoll_user_data[socket_id].mmap_buffer.bytes_size) { epoll_user_data[socket_id].mmap_buffer.bytes_size += mmap_grow_size; }
        SHF_DEBUG("%f %6lu=tid socket_id #%d of %u mmap_buffer[+%u = %u bytes] for appending +%u = %u bytes\n", get_time_in_seconds(), gettid(), 1 + socket_id, EPOLL_LISTENING_PORTS, epoll_user_data[socket_id].mmap_buffer.bytes_size - old_byte_size, epoll_user_data[socket_id].mmap_buffer.bytes_size, src_len, src_len + epoll_user_data[socket_id].mmap_buffer.bytes_used);
        epoll_user_data[socket_id].mmap_buffer.bytes = mremap(epoll_user_data[socket_id].mmap_buffer.bytes, old_byte_size, epoll_user_data[socket_id].mmap_buffer.bytes_size, MREMAP_MAYMOVE); SHF_ASSERT_STRERROR(MAP_FAILED != epoll_user_data[socket_id].mmap_buffer.bytes, "%p=mremap()", epoll_user_data[socket_id].mmap_buffer.bytes);
    }
    else {
        SHF_DEBUG("%f %6lu=tid socket_id #%d of %u mmap_buffer[+%u = %u bytes] for appending +%u = %u bytes\n", get_time_in_seconds(), gettid(), 1 + socket_id, EPOLL_LISTENING_PORTS, 0, epoll_user_data[socket_id].mmap_buffer.bytes_size, src_len, src_len + epoll_user_data[socket_id].mmap_buffer.bytes_used);
    }
    char * dst_bytes = SHF_CAST(char *, epoll_user_data[socket_id].mmap_buffer.bytes);
    memcpy(&dst_bytes[epoll_user_data[socket_id].mmap_buffer.bytes_used], src_bytes, src_len);
    epoll_user_data[socket_id].mmap_buffer.bytes_used += src_len;
 }

 void tftp_send_ack(int socket_id)
 {
    char ack_packet[4];
    int socket_fd_out = epoll_user_data[socket_id].fd;

    uint16_t ack   = TFTP_PACKET_TYPE_ACK                 ; SHF_U08_AT(ack_packet, 0) = (ack   >> 8) & 0xff; SHF_U08_AT(ack_packet, 1) = (ack   >> 0) & 0xff; //fixme: is there a way to arrange the byte order more elegantly?
    uint16_t block = epoll_user_data[socket_id].tftp_block; SHF_U08_AT(ack_packet, 2) = (block >> 8) & 0xff; SHF_U08_AT(ack_packet, 3) = (block >> 0) & 0xff;

    SHF_DEBUG("%f %6lu=tid send ack udp packet on listening socket_id #%u of %u on port %u with fd %d; sent %ld bytes to peer %s:%u for file '%.*s'\n", get_time_in_seconds(), gettid(), 1 + socket_id, EPOLL_LISTENING_PORTS, epoll_user_data[socket_id].hostshort, socket_fd_out, sizeof(ack_packet), epoll_user_data[socket_id].udp_peer_ipv4, epoll_user_data[socket_id].udp_peer_port, epoll_user_data[socket_id].file_name_len, &epoll_user_data[socket_id].file_name[0]);

    int flags = 0;
    ssize_t sent_bytes = sendto(socket_fd_out, &ack_packet[0], sizeof(ack_packet), flags, (struct sockaddr *) &epoll_user_data[socket_id].udp_peer_address, epoll_user_data[socket_id].udp_peer_address_len);
    if (sent_bytes < 0) {
        if ((EAGAIN      == errno)
        ||  (EWOULDBLOCK == errno)) {
            SHF_ASSERT_STRERROR(0, "%ld=sendto(); unexpected blocking sending to fd %d; todo: implement epoll wait to unblock\n", sent_bytes, socket_fd_out);
        }
        SHF_ASSERT_STRERROR(0, "%ld=sendto(); unexpected error sending to fd %d\n", sent_bytes, socket_fd_out);
    }
    else if (sent_bytes == sizeof(ack_packet)) { } // sent packet as expected
    else                                       { SHF_ASSERT_STRERROR(0, "%ld=sendto(); unexpected to send %lu bytes sending to fd %d\n", sent_bytes, sizeof(ack_packet), socket_fd_out); }
 }

 void* tftp_epoll_thread(void *arg)
 {
    int    socket_fd;
    struct epoll_event ev;
    struct epoll_event events[EPOLL_EVENTS_MAX];
    int    epoll_fd = epoll_create1(0); SHF_ASSERT_STRERROR(-1 != epoll_fd, "%d=epoll_fd=epoll_create1() should not be -1\n", epoll_fd); // https://stackoverflow.com/questions/10011252/whats-the-advantage-of-using-epoll-create1-instead-of-epoll-create

    socket_id_list_init(&socket_id_list_unused);
    socket_id_list_init(&socket_id_list_ready);

    for(int socket_id = 0; socket_id < EPOLL_LISTENING_PORTS; socket_id ++) {
        socket_fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); SHF_ASSERT_STRERROR(-1 != socket_fd, "%d=socket_fd=socket() should not be -1\n", socket_fd);

        int get_flag = fcntl(socket_fd, F_GETFL, 0                    ); SHF_ASSERT_STRERROR(-1 != get_flag, "%d=get_flag=fcntl() should not be -1\n", get_flag);
        int set_flag = fcntl(socket_fd, F_SETFL, get_flag | O_NONBLOCK); SHF_ASSERT_STRERROR(-1 != set_flag, "%d=set_flag=fcntl() should not be -1\n", set_flag);

        { int enable = 1; int result = setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(enable)); SHF_ASSERT_STRERROR(-1 != result, "%d=result=setsockopt() should not be -1\n", result); }

        epoll_user_data[socket_id].fd                     = socket_fd;
        epoll_user_data[socket_id].mmap_buffer.bytes      = NULL;
        epoll_user_data[socket_id].mmap_buffer.bytes_size = 0;
        epoll_user_data[socket_id].mmap_buffer.bytes_used = 0;
        epoll_user_data[socket_id].socket_status          = SOCKET_ID_STATUS_UNUSED;
        epoll_user_data[socket_id].hostshort              = (0 == socket_id) ? EPOLL_LISTENING_PORT_TFTP : 0 /* kernel chooses port */ ;

        struct sockaddr_in address;
        socklen_t          address_len = sizeof(address);
        memset(&address, 0, address_len);
        address.sin_family      = AF_INET;
        address.sin_addr.s_addr = htonl(INADDR_ANY);
        address.sin_port        = htons(epoll_user_data[socket_id].hostshort);
        { int result = bind(socket_fd, (struct sockaddr *) &address, address_len); SHF_ASSERT_STRERROR(-1 != result, "%d=result=bind() should not be -1\n", result); }

        if (0 == epoll_user_data[socket_id].hostshort) {
            socket_id_list_push(&socket_id_list_unused, socket_id);

            struct sockaddr_in sin;
            int len = sizeof(sin);
            int result = getsockname(socket_fd, (struct sockaddr *)&sin, &len); SHF_ASSERT_STRERROR(-1 != result, "%d=result=getsockname() should not be -1\n", result);
            epoll_user_data[socket_id].hostshort = ntohs(sin.sin_port); // remember kernel chosen port
        }
        printf("%f %6lu=tid socket_id #%u of %u listening on port %u with fd %d%s\n", get_time_in_seconds(), gettid(), 1 + socket_id, EPOLL_LISTENING_PORTS, epoll_user_data[socket_id].hostshort, socket_fd, 0 == socket_id ? " <- tfpt main port" : "");

        ev.data.u64 = socket_id;
        ev.events = EPOLLIN | EPOLLET; // events are read only and edge-triggered
        int ctl_fd  = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, socket_fd, &ev); SHF_ASSERT_STRERROR(-1 != ctl_fd, "%d=ctl_fd=bind() should not be -1\n", ctl_fd);
    } // for(int socket_id ..

    printf("%f %6lu=tid entering epoll event loop\n", get_time_in_seconds(), gettid());
    while(1) {
        int num_events = epoll_wait(epoll_fd, events, EPOLL_EVENTS_MAX, -1 /* wait indefinitely */);
        SHF_DEBUG("%f %6lu=tid looping through %u epoll event(s)\n", get_time_in_seconds(), gettid(), num_events);
        for (int i = 0; i < num_events; i++) {
            if (  (events[i].events & EPOLLERR)
            ||    (events[i].events & EPOLLHUP)
            ||  (!(events[i].events & EPOLLIN ))) {
                SHF_ASSERT(0, "%u=events[i].events=EPOLLERR|EPOLLHUP|!EPOLLIN; unexpected error occurred on fd %d\n", events[i].events, events[i].data.fd);
            }
            else if (events[i].events & EPOLLIN) {
                // come here if data on the fd waiting to be read, so read it now else no further notification due to edge-triggered mode
                int socket_id = events[i].data.u64;
                SHF_ASSERT(socket_id >= 0                    , "%d=socket_id should be >= 0\n", socket_id);
                SHF_ASSERT(socket_id <  EPOLL_LISTENING_PORTS, "%d=socket_id should be < %u\n", socket_id, EPOLL_LISTENING_PORTS);
                socket_fd = epoll_user_data[socket_id].fd;
                while(1) { // greedy UDP read ... fixme:only useful for tftp main port, so only be greedy on the main port; saves unnecessary recvfrom() on other ports
                    // read a udp packet
                    struct sockaddr_in udp_peer_address;
                    int                udp_peer_address_len = sizeof(udp_peer_address);
                    int udp_buf_used = recvfrom(socket_fd, udp_buf, UDP_BUF_SIZE, 0, (struct sockaddr *)&udp_peer_address, (socklen_t *)&udp_peer_address_len);

                    if (udp_buf_used < 0) {
                        if ((EAGAIN      == errno)
                        ||  (EWOULDBLOCK == errno)) {
                            break; // no more packets to read on this socket
                        }
                        SHF_ASSERT_STRERROR(0, "%d=recvfrom(); unexpected error reading fd %d\n", udp_buf_used, events[i].data.fd);
                    }
                    else if(0 == udp_buf_used) { printf("%f %6lu=tid WARNING: %d=recvfrom(); unexpected value reading fd %d; ignoring packet\n", get_time_in_seconds(), gettid(), udp_buf_used, events[i].data.fd); }
                    else {
                        // determine the peer ip:port that sent the packet
                        SHF_ASSERT(AF_INET == udp_peer_address.sin_family, "%u=udp_peer_address.sin_family should be %u=AF_INET\n", udp_peer_address.sin_family, AF_INET);
                        int  udp_peer_port = ntohs(udp_peer_address.sin_port);
                        char udp_peer_ipv4[INET_ADDRSTRLEN];
                        inet_ntop(AF_INET, &udp_peer_address.sin_addr, udp_peer_ipv4, INET_ADDRSTRLEN);
                        SHF_DEBUG("%f %6lu=tid event %d of %d: read udp packet on listening socket_id #%u of %u on port %u with fd %d; read %d bytes from peer %s:%u\n", get_time_in_seconds(), gettid(), 1 + i, num_events, 1 + socket_id, EPOLL_LISTENING_PORTS, epoll_user_data[socket_id].hostshort, socket_fd, udp_buf_used, udp_peer_ipv4, udp_peer_port);
                        hex_dump("udp_buf", udp_buf, udp_buf_used);

                        if (0 == socket_id) {
                            // come here if received udp packet on tftp main port
                            int min_tftp_main_port_packet_size = 2 /* size of tftp packet type length */ + 1 /* smallest file name length */ + 1 /* zero termination */ + 4 /* shortest mode */ + 1 /* zero termination */;
                            if       (udp_buf_used < min_tftp_main_port_packet_size        ) { printf("%f %6lu=tid WARNING: packet size %u < minimum size %u"       "; dropping packet, ignoring request\n", get_time_in_seconds(), gettid(), udp_buf_used, min_tftp_main_port_packet_size); }
                            else if  ( 0  !=         SHF_U08_AT(udp_buf, udp_buf_used - 1) ) { printf("%f %6lu=tid WARNING: packet is not zero terminated"          "; dropping packet, ignoring request\n", get_time_in_seconds(), gettid()); }
                            else if  ('i' == tolower(SHF_U08_AT(udp_buf, udp_buf_used - 2))) { printf("%f %6lu=tid WARNING: packet tftp mode netascii" "not supported; dropping packet, ignoring request\n", get_time_in_seconds(), gettid()); }
                            else if  ('l' == tolower(SHF_U08_AT(udp_buf, udp_buf_used - 2))) { printf("%f %6lu=tid WARNING: packet tftp mode mail"     "not supported; dropping packet, ignoring request\n", get_time_in_seconds(), gettid()); }
                            else if (( 0  ==         SHF_U08_AT(udp_buf, udp_buf_used - 7) )
                            &&       ('o' == tolower(SHF_U08_AT(udp_buf, udp_buf_used - 6)))
                            &&       ('c' == tolower(SHF_U08_AT(udp_buf, udp_buf_used - 5)))
                            &&       ('t' == tolower(SHF_U08_AT(udp_buf, udp_buf_used - 4)))
                            &&       ('e' == tolower(SHF_U08_AT(udp_buf, udp_buf_used - 3)))
                            &&       ('t' == tolower(SHF_U08_AT(udp_buf, udp_buf_used - 2)))) {
                                // come here if tftp main port packet is octet mode
                                int      file_name_zero_termination_pos = udp_buf_used - 7;
                                int      file_name_len = file_name_zero_termination_pos - 2 + 1;
                                uint16_t tftp_packet_type = (SHF_U08_AT(udp_buf, 0) << 8) + SHF_U08_AT(udp_buf, 1);
                                if      ((TFTP_PACKET_TYPE_RRQ != tftp_packet_type)
                                &&       (TFTP_PACKET_TYPE_WRQ != tftp_packet_type)) { printf("%f %6lu=tid WARNING: unknown tfpt packet type %u; dropping packet, ignoring request\n", get_time_in_seconds(), gettid(), tftp_packet_type); }
                                else if (file_name_zero_termination_pos <= 2       ) { printf("%f %6lu=tid WARNING: bad filename"             "; dropping packet, ignoring request\n", get_time_in_seconds(), gettid()); }
                                else if (file_name_len > TFTP_FILE_NAME_LEN_MAX    ) { printf("%f %6lu=tid WARNING: filename too long"        "; dropping packet, ignoring request\n", get_time_in_seconds(), gettid()); }
                                else {
                                    //fixme: sanity check filename, e.g. only ascii chars etc
                                    // come here if RRQ or WRQ in octet mode
                                    int socket_id_2_use = socket_id_list_pop(&socket_id_list_unused);
                                    if (-1 == socket_id_2_use) { printf("%f %6lu=tid WARNING: no free socket_id to service tftp request; dropping packet, ignoring request\n", get_time_in_seconds(), gettid());}
                                    else {
                                        // come here if RRQ or WRQ in octet mode and free socket_id
                                        epoll_user_data[socket_id_2_use].timestamp_1st = get_time_in_seconds();
                                        epoll_user_data[socket_id_2_use].tftp_block    = 0;
                                        epoll_user_data[socket_id_2_use].file_name_len = file_name_len;
                                        memcpy(&epoll_user_data[socket_id_2_use].file_name[0], &SHF_U08_AT(udp_buf, 2), file_name_len);
                                        if (TFTP_PACKET_TYPE_RRQ == tftp_packet_type) { printf("%f %6lu=tid WARNING: tftp packet type RRQ not supported; dropping packet, ignoring request\n", get_time_in_seconds(), gettid()); }
                                        else {
                                            // come here to process TFTP_PACKET_TYPE_WRQ
                                            epoll_user_data[socket_id_2_use].udp_peer_address_len = udp_peer_address_len                   ; memcpy(&epoll_user_data[socket_id_2_use].udp_peer_address, &udp_peer_address, udp_peer_address_len); // copy peer address to socket_id_2_use state
                                            epoll_user_data[socket_id_2_use].udp_peer_port        = udp_peer_port                          ; memcpy(&epoll_user_data[socket_id_2_use].udp_peer_ipv4   , &udp_peer_ipv4   , INET_ADDRSTRLEN     ); // copy peer ipv4 to socket_id_2_use state
                                            epoll_user_data[socket_id_2_use].socket_status        = SOCKET_ID_STATUS_WAITING_FOR_PEER_WRITE;
                                            tftp_send_ack(socket_id_2_use);
                                        }
                                    }
                                }
                            }
                            else { printf("%f %6lu=tid WARNING: packet tftp mode not identified as netascii, mail, or octet; dropping packet, ignoring request\n", get_time_in_seconds(), gettid()); }
                        }
                        else {
                            // come here if received udp packet on a tftp slave port
                            if      (epoll_user_data[socket_id].socket_status                    == SOCKET_ID_STATUS_UNUSED         )  { printf("%f %6lu=tid WARNING: packet unexpected"                                 "; dropping packet\n", get_time_in_seconds(), gettid()); }
                            else if((epoll_user_data[socket_id].udp_peer_address.sin_addr.s_addr != udp_peer_address.sin_addr.s_addr)
                            ||      (epoll_user_data[socket_id].udp_peer_address.sin_port        != udp_peer_address.sin_port       )) { printf("%f %6lu=tid WARNING: expected packet from %s:%u but got packet from %s:%u; dropping packet\n", get_time_in_seconds(), gettid(), epoll_user_data[socket_id].udp_peer_ipv4, epoll_user_data[socket_id].udp_peer_port, udp_peer_ipv4, udp_peer_port); }
                            else {
                                // come here if received udp packet on a tftp slave port from expected peer
                                int min_tftp_slave_port_packet_size = 2 /* size of tftp packet type length */ + 2 /* block size */;
                                if (udp_buf_used < min_tftp_slave_port_packet_size) { printf("%f %6lu=tid WARNING: packet size %u < minimum size %u; dropping packet, ignoring request\n", get_time_in_seconds(), gettid(), udp_buf_used, min_tftp_slave_port_packet_size); }
                                else {
                                    // come here if received udp packet on a tftp slave port from expected peer and big enough
                                    uint16_t tftp_packet_type = (SHF_U08_AT(udp_buf, 0) << 8) + SHF_U08_AT(udp_buf, 1);
                                    uint16_t tftp_block       = (SHF_U08_AT(udp_buf, 2) << 8) + SHF_U08_AT(udp_buf, 3);
                                    if      (tftp_packet_type != TFTP_PACKET_TYPE_DATA                    ) { printf("%f %6lu=tid WARNING: expected tftp type"  " 0x%04x but got 0x%04x; dropping packet, ignoring request\n", get_time_in_seconds(), gettid(), TFTP_PACKET_TYPE_DATA                    , tftp_packet_type); }
                                    else if (tftp_block       != 1 + epoll_user_data[socket_id].tftp_block) { printf("%f %6lu=tid WARNING: expected tftp block" " 0x%04x but got 0x%04x; dropping packet, ignoring request\n", get_time_in_seconds(), gettid(), 1 + epoll_user_data[socket_id].tftp_block, tftp_block      ); }
                                    // come here if received udp packet on a tftp slave port from expected peer and big enough and expected tftp type and expected block
                                    epoll_user_data[socket_id].tftp_block ++; //fixme: what happens with the tftp block if it wraps?
                                    tftp_send_ack(socket_id);
                                    int block_length = udp_buf_used - 2 /* tftp type */ - 2 /* tftp block */; //fimme: refactor "tftp type" to "tftp opcode"
                                    if (0 == block_length) {
                                        double timestamp_end = get_time_in_seconds();
                                        double elapsed_seconds = timestamp_end - epoll_user_data[socket_id].timestamp_1st;
                                        printf("%f %6lu=tid socket_id #%u of %u packet from expected peer %s:%u with tftp type 0x%04x and block 0x%04x; transfer complete: %'u bytes in %f seconds or %'.1f PPS for file '%.*s'\n", get_time_in_seconds(), gettid(), 1 + socket_id, EPOLL_LISTENING_PORTS, udp_peer_ipv4, udp_peer_port, tftp_packet_type, tftp_block, epoll_user_data[socket_id].mmap_buffer.bytes_used, elapsed_seconds, tftp_block / elapsed_seconds, epoll_user_data[socket_id].file_name_len, &epoll_user_data[socket_id].file_name[0]);
                                        //fixme: consider more efficient multi-threaded logging because printf() contains a lock and will slow performance when many files per second
                                        epoll_user_data[socket_id].socket_status = SOCKET_ID_STATUS_UNUSED; //fixme: make .socket_status and socket_id_lists more seemless and elegant
                                        socket_id_list_push(&socket_id_list_ready, socket_id); // push socket_id onto ready list
                                    }
                                    else {
                                        SHF_DEBUG("%f %6lu=tid socket_id #%u of %u packet from expected peer %s:%u with tftp type 0x%04x and block 0x%04x; appending to mmap\n", get_time_in_seconds(), gettid(), 1 + socket_id, EPOLL_LISTENING_PORTS, udp_peer_ipv4, udp_peer_port, tftp_packet_type, tftp_block);
                                        mmap_buffer_append(socket_id, &SHF_U08_AT(udp_buf, 4), block_length); //fixme: replace hard-coded 4 with something more elegant
                                    }
                                }
                            }
                        }
                    }
                } // while(1)
            } // if EPOLLIN
        } // for (int i
    } // while(1)
    //fixme: loop through and free all mmaps
    //fixme: loop through and close all open sockets, and epoll_fd
 } // tftp_epoll_thread()

 void tftp_init(void)
 {
    setlocale(LC_NUMERIC, ""); // so that printf "%'u" displays numeric commas

    pthread_t tid;
    int result = pthread_create(&tid, NULL, &tftp_epoll_thread, NULL); SHF_ASSERT_STRERROR(0 == result, "%d=result=pthread_create() should be 0\n", result);
    printf("%f %6lu=tid tftp epoll thread created\n", get_time_in_seconds(), gettid());
 }

 //fixme: horrible API for passing C data to higher level caller such as Golang; maybe better to back the 'files' on /dev/shm ? How to notify in Golang?
 int    ready_file_name_socket_id = 0;
 char * ready_file_name_pointer;
 int    ready_file_name_len;
 char * ready_file_body_pointer;
 int    ready_file_body_len;

 int tftp_file_ready(void)
 {
    SHF_ASSERT(0 == ready_file_name_socket_id, "%d=ready_file_name_socket_id must call tftp_file_ready_recycle() after tftp_file_ready()\n", ready_file_name_socket_id);
    if (socket_id_list_ready.total > 0) {
        int socket_id             = socket_id_list_pop(&socket_id_list_ready);
        ready_file_name_socket_id = socket_id;
        ready_file_name_pointer   = &epoll_user_data[socket_id].file_name[0];
        ready_file_name_len       = epoll_user_data[socket_id].file_name_len;
        ready_file_body_pointer   = epoll_user_data[socket_id].mmap_buffer.bytes;
        ready_file_body_len       = epoll_user_data[socket_id].mmap_buffer.bytes_used;
        return 1;
    }
    return 0;
 }

 void tftp_file_ready_recycle(void)
 {
    SHF_ASSERT(0 != ready_file_name_socket_id, "%d=ready_file_name_socket_id must call tftp_file_ready() after tftp_file_ready_recycle()\n", ready_file_name_socket_id);
    int socket_id = ready_file_name_socket_id;
    epoll_user_data[socket_id].mmap_buffer.bytes_used = 0; // truncate mmap buffer for this socket_id
    socket_id_list_push(&socket_id_list_unused, socket_id); // allow socket_id to be used by next tftp client
    ready_file_name_socket_id = 0;
 }
diff --git a/shftftpd.go b/shftftpd.go
 package main
 /*
 #include "shftftpd.c"
 */
 import "C"
 import "fmt"
 import "time"
 import "unsafe"
 import "syscall"
 // go build -x shftftpd.go && ./shftftpd
 func main() {
    fmt.Printf("%f %6d=tid go: start c tftpd\n", C.get_time_in_seconds(), syscall.Gettid())
    C.tftp_init();
    uniqueFiles := make(map[string][]byte)
    for {
        if (1 == C.tftp_file_ready()) { //fixme: how to make C return a bool for nicer looking code?
            var start time.Time = time.Now() //fixme: why does coping from c to go take so long?
            readyFileName := C.GoStringN(               C.ready_file_name_pointer , C.ready_file_name_len)
            readyFileBody := C.GoBytes  (unsafe.Pointer(C.ready_file_body_pointer), C.ready_file_body_len)
            uniqueFiles[readyFileName] = readyFileBody
            fmt.Printf("%f %6d=tid go: discovered file with %d bytes ready '%s', %d unique files so far (copied from c to go map in %f ms)\n", C.get_time_in_seconds(), syscall.Gettid(), len(readyFileBody), readyFileName, len(uniqueFiles), time.Now().Sub(start).Seconds() * 1000)
            C.tftp_file_ready_recycle();
        } else {
            time.Sleep(1 * time.Millisecond)
        }
    }
 }
	#include "shftftpd.c"

	int main(int argc, char * argv)
	{
	tftp_init();

	while(1) {
	if (socket_id_list_ready.total > 0) {
	while (socket_id_list_ready.total > 0) {
	int socket_id = socket_id_list_pop(&socket_id_list_ready);
	printf("%f %6lu=tid socket_id #%d of %u has transferred a %'u byte file '%.*s'\n", get_time_in_seconds(), gettid(), 1 + socket_id, EPOLL_LISTENING_PORTS, epoll_user_data[socket_id].mmap_buffer.bytes_used, epoll_user_data[socket_id].file_name_len, &epoll_user_data[socket_id].file_name[0]);
	//fixme: do something with transferred file bytes
	epoll_user_data[socket_id].mmap_buffer.bytes_used = 0; // truncate mmap buffer for this socket_id
	socket_id_list_push(&socket_id_list_unused, socket_id); // allow socket_id to be used by next tftp client
	}
	socket_id_list_ready.total = 0;
	}
	else {
	sleep_ms(2); // poll for ready tftp files
	}
	}

	return EXIT_SUCCESS;
	} // main()
	package main
	/*
	#include "shftftpd.c"
	*/
	import "C"
	import "fmt"
	import "time"
	import "unsafe"
	import "syscall"
	// go build -x shftftpd.go && ./shftftpd
	func main() {
	fmt.Printf("%f %6d=tid go: start c tftpd\n", C.get_time_in_seconds(), syscall.Gettid())
	C.tftp_init();
	uniqueFiles := make(map[string][]byte)
	for {
	if (1 == C.tftp_file_ready()) { //fixme: how to make C return a bool for nicer looking code?
	var start time.Time = time.Now() //fixme: why does coping from c to go take so long?
	readyFileName := C.GoStringN( C.ready_file_name_pointer , C.ready_file_name_len)
	readyFileBody := C.GoBytes (unsafe.Pointer(C.ready_file_body_pointer), C.ready_file_body_len)
	uniqueFiles[readyFileName] = readyFileBody
	fmt.Printf("%f %6d=tid go: discovered file with %d bytes ready '%s', %d unique files so far (copied from c to go map in %f ms)\n", C.get_time_in_seconds(), syscall.Gettid(), len(readyFileBody), readyFileName, len(uniqueFiles), time.Now().Sub(start).Seconds() * 1000)
	C.tftp_file_ready_recycle();
	} else {
	time.Sleep(1 * time.Millisecond)
	}
	}
	}