A Lua AFL integration using the debug hook functionality which fires as Lua traverses lines

afl-fuzz.c

// Using the approach of afl-python to make a
// Lua fuzzer.
// Build with "gcc -I/usr/include/lua5.3/ -L/usr/local/lib -llua5.3 -rdynamic afl-fuzz.c"
// (or whatever works on your platform).
//
// Write a script which has a global function "fuzz" which reads all of stdin and processes it
// to exercise some code in which you'd like to find logic bugs.

#include <assert.h>
#include <fcntl.h>
#include <lauxlib.h>
#include <lua.h>
#include <lualib.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/shm.h>
#include <sys/wait.h>
#include <unistd.h>



// The presence of this string is enough to allow
// AFL fuzz to run without using the env variable
// AFL_SKIP_BIN_CHECK
const char *SHM_ENV  = "__AFL_SHM_ID";
const char *NOFORK   = "AFL_NO_FORKSRV";


const int afl_read_fd = 198;
const int afl_write_fd = afl_read_fd + 1;

static unsigned char *afl_shm;
static size_t afl_shm_size = 1 << 16;


static int shm_init() {
    const char *shm = getenv(SHM_ENV);
    assert(shm);
    afl_shm = shmat(atoi(shm), NULL, 0);
    assert(afl_shm > 0);
    return 0;
}

static int fork_write(int pid) {
    const int ok = (4 == write(afl_write_fd, &pid, 4));
    assert(ok);
    return 0;
}

static int fork_read() {
    int tmp;
    const int ok = (4 == read(afl_read_fd, &tmp, 4));
    assert(ok);
    return 0;
}

static int fork_close() {
    close(afl_read_fd);
    close(afl_write_fd);
    return 0;
}

static int run_target(lua_State *L) {
    if(lua_pcall(L, 0, 0, 0)) {
        abort();
    }
    return 0;
}

/**
 * From afl-python
 * https://github.com/jwilk/python-afl/blob/8df6bfefac5de78761254bf5d7724e0a52d254f5/afl.pyx#L74-L87
 */
#define LHASH_INIT       0x811C9DC5
#define LHASH_MAGIC_MULT 0x01000193
#define LHASH_NEXT(x)    h = ((h ^ (unsigned char)(x)) * LHASH_MAGIC_MULT)

static inline unsigned int lhash(const char *key, size_t offset) {
    const char *const last = &key[strlen(key) - 1];
    uint32_t h = LHASH_INIT;
    while (key <= last)               LHASH_NEXT(*key++);
    for (; offset != 0; offset >>= 8) LHASH_NEXT(offset);
    return h;
}

static unsigned int current_location;

static void hook(lua_State *L, lua_Debug *ar) {
    lua_getinfo(L, "Sl", ar);
    if (ar && ar->source && ar->currentline) {
        const unsigned int new_location = lhash(ar->source, ar->currentline) % afl_shm_size;
        afl_shm[current_location ^ new_location] += 1;
        current_location = new_location / 2;
    }
}

int main(int argc, const char **argv) {
    shm_init();

    assert(argc > 1);
    const char *fuzz_target = argv[1];

    lua_State *L = luaL_newstate();
    assert(L);

    luaL_openlibs(L);

    lua_sethook(L, hook, LUA_MASKLINE, 0);


    int rc = luaL_dofile(L, fuzz_target);
    assert(rc == 0);

    lua_getglobal(L, "fuzz");
    if (!lua_isfunction(L, -1)) {
        assert(0);
    }


    if (getenv(NOFORK)) {
        run_target(L);
        return 0;
    }

    fork_write(0); // let AFL know we're here

    while (1) {
        fork_read(); // discard report from AFL
        pid_t child = fork();
        if (child == 0) {
            fork_close();
            run_target(L);
            return 0;
        }
        fork_write(child);
        int status = 0;
        rc = wait(&status);
        fork_write(status);
    }

    return 0;
}