Demo of a simple custom virtual machine. Presentation available at https://www.slideshare.net/BartoszSypytkowski1/virtual-machines-how-they-work/

main.c

#include <stdio.h>
#include <stdlib.h>

// stack will have fixed size
#define STACK_SIZE 1024
#define DEBUG_ENABLED 1
#define DBG if(DEBUG_ENABLED) printf
#define PUSH(vm, v) vm->stack[++vm->sp] = v // push value on top of the stack
#define POP(vm)     vm->stack[vm->sp--]     // pop value from top of the stack
#define NEXT(vm)   vm->code[vm->pc++]      // get next bytecode

enum {
    Constant = 0,       // push constant integer
    Add = 1,            // int add
    Sub = 2,            // int sub
    Mul = 3,            // int mul
    LessThan = 4,       // int less than
    Equal = 5,          // int equal
    Jump = 6,           // branch
    JumpIfTrue = 7,     // branch if true
    JumpIfFalse = 8,    // branch if false
    LoadArgument = 9,   // load function argument at position given in next opcode
    Print = 10,         // print value on top of the stack
    Call = 11,          // call procedure
    Return = 12,        // return from procedure
    Halt = 13           // stop program
};

typedef struct {
    int* code;      // array od byte codes to be executed
    int* stack;     // virtual stack
    int pc;         // program counter (aka. IP - instruction pointer)
    int sp;         // stack pointer
    int fp;         // frame pointer (for local scope)
} VM;

VM* vm_new(int* code,       // pointer to table containing a bytecode to be executed
          int pc) {         // address of instruction to be invoked as first one - entrypoint/main func
    VM* vm = (VM*)malloc(sizeof(VM));
    vm->code = code;
    vm->pc = pc;
    vm->fp = 0;
    vm->sp = -1;
    vm->stack = (int*)malloc(sizeof(int) * STACK_SIZE);

    return vm;
}

void vm_delete(VM* vm){
    free(vm->stack);
    free(vm);
}

void op_constant(VM* vm) {
    int v = NEXT(vm);
    PUSH(vm, v);
    DBG("Constant %d\n", v);
}

void op_add(VM* vm) {
    int b = POP(vm);
    int a = POP(vm);
    int rval = a + b;
    PUSH(vm, rval);
    DBG("Add %d + %d => %d\n", a, b, rval);
}

void op_lt(VM* vm) {
    int b = POP(vm);
    int a = POP(vm);
    int rval = (a < b) ? 1 : 0;
    PUSH(vm, rval);
    DBG("LessThan %d < %d => %d\n", a, b, rval);
}

void op_sub(VM* vm) {
    int b = POP(vm);
    int a = POP(vm);
    int rval = a - b;
    PUSH(vm, rval);
    DBG("Sub %d - %d => %d\n", a, b, rval);
}

void op_mul(VM* vm) {
    int b = POP(vm);
    int a = POP(vm);
    int rval = a * b;
    PUSH(vm, rval);
    DBG("Mul %d * %d => %d\n", a, b, rval);
}

void op_eq(VM* vm) {
    int b = POP(vm);
    int a = POP(vm);
    int rval = (a<b) ? 1 : 0;
    PUSH(vm, rval);
    DBG("Equal %d == %d => %d\n", a, b, rval);
}

void op_jmp(VM* vm) {
    int addr = NEXT(vm);
    DBG("%d:\tJump %d\n", addr);
    vm->pc = addr;
}

void op_jmp_eq(VM* vm) {
    int addr = NEXT(vm);
    int v = POP(vm);
    if(v) {
        vm->pc = addr;
    }
    DBG("JumpIfTrue %d => %d\n", v, addr);
}

void op_jmp_ne(VM* vm) {
    int addr = NEXT(vm);
    int v = POP(vm);
    if(!v) {
        vm->pc = addr;
    }
    DBG("JumpIfFalse %d => %d\n", v, addr);
}

void op_call(VM* vm) {
    // we expect all args to be on the stack
    int addr = NEXT(vm);
    int argc = NEXT(vm);
    PUSH(vm, argc);
    PUSH(vm, vm->fp);
    PUSH(vm, vm->pc);
    DBG("Call (addr:%d, argc:%d, fp:%d)\n", addr, argc, vm->fp);
    vm->fp = vm->sp;
    vm->pc = addr;
}

void op_lda(VM* vm) {
    int offset = NEXT(vm) + 3;
    int v = vm->stack[vm->fp-offset];
    PUSH(vm, v);
    printf("LoadArgument %d => %d\n", offset-3, v);
}

void op_ret(VM* vm) {
    int rval = POP(vm);
    vm->pc = POP(vm);
    vm->fp = POP(vm);
    int argc = POP(vm);
    vm->sp -= argc;
    PUSH(vm, rval);
    DBG("Return\n");
}

void op_print(VM* vm) {
    printf("PRINTED: %d\n", POP(vm));
}

void (*op_handle[])(VM*) = {
        op_constant,    // Constant = 0,
        op_add,         // Add = 1,
        op_sub,         // Sub = 2,
        op_mul,         // Mul = 3,
        op_lt,          // LessThan = 4,
        op_eq,          // Equal = 5,
        op_jmp,         // Jump = 6,
        op_jmp_eq,      // JumpIfTrue = 7,
        op_jmp_ne,      // JumpIfFalse = 8,
        op_lda,         // LoadArgument = 9,
        op_print,       // Print = 10,
        op_call,        // Call = 11,
        op_ret          // Return = 14,
};

void vm_run(VM* vm){
    do{
        int opcode = NEXT(vm);        // fetch next instruction

        if (opcode == Halt) {
            DBG("Halt\n");
            return;
        }

        op_handle[opcode](vm);
    }while(1);
}

int main() {
    const int fib = 0;  // address of the fibonacci procedure
    int program[] = {
            // int fib(n) {
            //     if(n == 0) return 0;
            LoadArgument, 0,       // 0 - load first function argument N
            Constant, 0,           // 2 - put 0
            Equal,                 // 4 - check equality: N == 0
            JumpIfFalse, 10,       // 5 - if they are NOT equal, goto 10
            Constant, 0,           // 7 - otherwise put 0
            Return,                // 9 - and return it
            //     if(n < 3) return 1;
            LoadArgument, 0,       // 10 - load last function argument N
            Constant, 3,           // 12 - put 3
            LessThan,              // 14 - check if 3 is less than N
            JumpIfFalse, 20,       // 15 - if 3 is NOT less than N, goto 20
            Constant, 1,           // 17 - otherwise put 1
            Return,                // 19 - and return it
            //     else return fib(n-1) + fib(n-2);
            LoadArgument, 0,       // 20 - load last function argument N
            Constant, 1,           // 22 - put 1
            Sub,                   // 24 - calculate: N-1, result is on the stack
            Call, fib, 1,          // 25 - call fib function with 1 arg. from the stack
            LoadArgument, 0,       // 28 - load N again
            Constant, 2,           // 30 - put 2
            Sub,                   // 32 - calculate: N-2, result is on the stack
            Call, fib, 1,          // 33 - call fib function with 1 arg. from the stack
            Add,                   // 36 - since 2 fibs pushed their ret values on the stack, just add them
            Return,                // 37 - return from procedure
            // entrypoint - main function
            Constant, 6,           // 38 - put 6
            Call, fib, 1,          // 40 - call function: fib(arg) where arg = 6;
            Print,                 // 43 - print result
            Halt                   // 44 - stop program
    };

    // initialize virtual machine
    VM* vm = vm_new(program,   // program to execute
                   38);        // start address of main function
    vm_run(vm);
    vm_delete(vm);
    return 0;
}