Jai implementation of Media Molecule's LBP serialization system

LBP_Serialization.jai

/*

How to use this:


 0. Define your data, and #load (*NOT* #import) this file.

        My_Data :: struct {
            lives    := 3;
            radians: float;
        }
        global_my_data: My_Data;

        #load "Serialization.jai";


 1. Define your Serializer with your data type and a version enum as parameters.

        My_Serializer :: Serializer(
            DATA    = My_Data,
            VERSION = enum {
                INITIAL;
            }
        );


 2. Write a serialize(using *My_Serializer, *My_Data) procedure, which invokes
    ADD(v, ..xyz), where v is the initial version from the enum, and xyz
    is one string for each member name.

        serialize :: (using serializer: *My_Serializer, data: *My_Data) {
            ADD(.INITIAL, "lives", "radians");
        }


 3. Use serialize_to_file()/serialize_from_file() when you want to save/load
    your data to/from disk. Both take your My_Serializer type as the first
    argument, a filename as the second argument, and a pointer to the My_Data
    that you want to (de)serialize.

        if ctrl_s_pressed()
            then serialize_to_file(My_Serializer, "data.dat", *global_my_data);

        if ctrl_l_pressed()
            then deserialize_from_file(My_Serializer, "data.dat", *global_my_data);


 4. When you want to add a member to My_Data, first add a new version to
    My_Serializer's VERSION enum, then add an ADD(v, m) line to your serialize()
    procedure.

        My_Data :: struct {
            lives      := 3;
            radians:   float;
            continues: int;
        }

        My_Serializer :: Serializer(
            DATA    = My_Data,
            VERSION = enum {
                INITIAL;
                ADDED_CONTINUES;
            }
        );

        serialize :: (using serializer: *My_Serializer, data: *My_Data) {
            ADD(.INITIAL, "lives", "radians");
            ADD(.ADDED_CONTINUES, "continues");
        }


 5. When you want to remove a member from My_Data, once again, add a new version
    to My_Serializer's VERSION enum, then find the ADD() line where the member
    was added, make note of the version it was added, and add a REM(v0, v1, m)
    line to your serialize() procedure, where v0 is the version it was added,
    and v1 is the version it was removed. Then, edit that ADD() line to remove
    the string for that member.

        My_Data :: struct {
            lives      := 3;
            continues: int;
        }

        My_Serializer :: Serializer(
            DATA    = My_Data,
            VERSION = enum {
                INITIAL;
                ADDED_CONTINUES;
                REMOVED_RADIANS;
            }
        );

        serialize :: (using serializer: *My_Serializer, data: *My_Data) {
            ADD(.INITIAL, "lives");
            ADD(.ADDED_CONTINUES, "continues");
            REM(.INITIAL, .REMOVED_RADIANS, "radians");
        }


 6. If you want My_Data to contain any other data structures you've written that
    may change in the future, write a serialize(*My_Serializer, *T) procedure
    for it as well, just like you did for My_Data:

        My_Data :: struct {
            lives      := 3;
            continues: int;
            other_data: Other_Data;
        }

        Other_Data :: struct {
            chosen_deity_name: string;
            blunders:          int;
        }

        My_Serializer :: Serializer(
            DATA    = My_Data,
            VERSION = enum {
                INITIAL;
                ADDED_CONTINUES;
                REMOVED_RADIANS;
                ADDED_OTHER_DATA;
            }
        );

        serialize :: (using serializer: *My_Serializer, data: *My_Data) {
            ADD(.INITIAL, "lives");
            ADD(.ADDED_CONTINUES, "continues");
            REM(.INITIAL, .REMOVED_RADIANS, "radians");
            ADD(.ADDED_OTHER_DATA, "other_data");
        }

        serialize :: (using serializer: *My_Serializer, data: *Other_Data) {
            ADD(.ADDED_OTHER_DATA, "chosen_deity_name", "blunders");
        }


 7. If you want to add a member to any of your data structures that you don't
    want to be serialized/deserialized, put a @NoSerialize note on it. (The note
    text can be customized by setting NO_SERIALIZE_NOTE in the Serializer if you
    so desire.)

        My_Data :: struct {
            lives      := 3;
            continues: int;
            other_data: Other_Data;
            some_table_that_we_fill_at_runtime: Table(int, string); @NoSerialize
        }

*/

Serializer :: struct(
    DEBUG             := false,
    DATA:             Type,             // your data
    VERSION:          Type,             // an enum
    NO_SERIALIZE_NOTE := "NoSerialize",
    PARAMETRIC_TYPES: ..Type            // any parametric struct Types you don't want auto-serialized
) {
    #assert(type_info(VERSION).type == .ENUM && type_info(VERSION).values.count > 0);
    DATA_VERSION_LATEST :: #run -> VERSION {
        ti := type_info(VERSION);
        return xx ti.values[ti.values.count-1];
    };

    data_version: VERSION;
    using endpoint: union mode: enum { FILE; DATA; } {
        .FILE,, file:      File;
        .DATA,, from_data: [] u8;
    }
    is_writing:   bool;

    ADD :: (data_version: VERSION, $field_names: ..string) #expand {
        if `serializer.data_version >= data_version {
            #insert -> string { builder: String_Builder;
                for field_names print(*builder, #string JAI
//╔════════════════════════════════════════════════════════════════════════════════════════════════╗
            serialize(`serializer, *`data.%);
//╚════════════════════════════════════════════════════════════════════════════════════════════════╝
                JAI, it);
                return builder_to_string(*builder);
            }
        }
    }

    REM :: (data_version_added: VERSION, data_version_removed: VERSION, $T: Type, field_name: string) {
        #insert #run tprint(#string JAI
//╔════════════════════════════════════════════════════════════════════════════════════════════════╗
        `%1: %2;
        if
            `serializer.data_version >= data_version_added   &&
            `serializer.data_version <  data_version_removed
        then serialize(`serializer, *`%1);
//╚════════════════════════════════════════════════════════════════════════════════════════════════╝
        JAI, field_name, T);
    }

    #if DEBUG
        then log_debug :: (message: string, args: ..Any) { log(message, ..args); }
        else log_debug :: (#discard message: string, #discard args: ..Any) {}
}

// Call this to serialize your data to a file
serialize_to_file :: ($SERIALIZER: Type, filename: string, data: *SERIALIZER.DATA) {
    using serializer := SERIALIZER.{
        data_version = SERIALIZER.DATA_VERSION_LATEST,
        mode         = .FILE,
        is_writing   = true,
    };
    log_debug("=== SERIALIZING ===");
    file=, file_open_success := file_open(filename, for_writing=true);
    assert(file_open_success);
    serialize(*serializer, *serializer.data_version);
    serialize(*serializer, data);
    file_close(*file);
    log_debug("=== DONE ===");
}

// Call this to deserialize your data from a file
deserialize_from_file :: ($SERIALIZER: Type, filename: string, data: *SERIALIZER.DATA) -> bool {
    using serializer := SERIALIZER.{
        mode       = .FILE,
        is_writing = false,
    };
    log_debug("=== DESERIALIZING ===");
    file=, file_open_success := file_open(filename, for_writing=false);
    assert(file_open_success);
    serialize(*serializer, *serializer.data_version);
    serialize(*serializer, data);
    file_close(*file);
    log_debug("=== DONE ===");
    return true;
}

// There is no serialize_to_data()

// Call this to deserialize your data from memory
deserialize_from_data :: ($SERIALIZER: Type, from_data: [] u8, data: *SERIALIZER.DATA) -> bool {
    using serializer := SERIALIZER.{
        mode       = .DATA,
        is_writing = false,
        from_data  = from_data,
    };
    log_debug("=== DESERIALIZING FROM DATA ===");
    serialize(*serializer, *serializer.data_version);
    serialize(*serializer, data);
    return true;
}

//--------------- Everything below is internal stuff for the serialization system ------------------

// Array view serializer
serialize :: (using serializer: *Serializer, data: *[] $T) {
    if is_writing {
        write(serializer, *data.count);
        write(serializer, data.data, data.count);
    } else {
        count := read(serializer, int);
        data.* = read(serializer, T, count);
    }
}

// string serializer
serialize :: (using serializer: *Serializer, data: *string) {
    serialize(serializer, data.(*[] u8));
}

// Scalar serializer
serialize :: (using serializer: *Serializer, data: *$T) #modify {
    ti := T.(*Type_Info_Variant);
    tt := ti.type;
    while true if tt == {
        case .BOOL;    #through;
        case .INTEGER; #through;
        case .ENUM;    #through;
        case .FLOAT;   return true;
        case .VARIANT; tt = ti.variant_of.type;
        case; return false;
    }
    return false;
} {
    log_debug("[SCALAR] %", T);
    if is_writing
        then write(serializer, data);
        else data.* = read(serializer, T);
}

// S128 serializer
serialize :: (using serializer: *Serializer, data: *$T) #modify {
    ti := T.(*Type_Info);
    while ti.type == .VARIANT ti = ti.(*Type_Info_Variant).variant_of;
    return ti == S128.(*Type_Info);
} {
    log_debug("[S128] %", T);
    if is_writing
        then write(serializer, data);
        else data.* = read(serializer, T);
}

// struct serializer
serialize :: (using serializer: *$SERIALIZER/Serializer, data: *$T) #modify {
    ti := T.(*Type_Info_Struct);
    if ti.runtime_size == 0 then return false,
        "T has a runtime size of zero."
    ;
    if ti.type != .STRUCT then return false,
        "T is not a struct type."
    ;
    if ti.textual_flags & .UNION then return false,
        "T is a union."
    ;
    if !ti.polymorph_source_struct then return true;

    tis := SERIALIZER.(*Type_Info_Struct);
    for member: tis.specified_parameters if member.name == "PARAMETRIC_TYPES" {
        for 0..member.type.(*Type_Info_Array).array_count-1 if ti == xx (tis.constant_storage.data + member.offset_into_constant_storage + size_of(Type)*it).(*Type).* then return false,
            "T is one of the types specified in PARAMETRIC_TYPES."
        ;
        break;
    }

    for get_common_default_module_parametric_structs() if ti.polymorph_source_struct == it then return false,
        "T is one of the common default module parametric struct types."
    ;

    return true;
} {
    log_debug("[STRUCT] %", T);
    #insert -> string { builder: String_Builder;
        for type_info(T).members {
            if (it.flags & .CONSTANT) || (it.flags & .OVERLAY) then continue;
            no_serialize := false;
            for it.notes if it == NO_SERIALIZE_NOTE { no_serialize = true; break; }
            if no_serialize then continue;
            print(*builder, #string JAI
//╔════════════════════════════════════════════════════════════════════════════════════════════════╗
    log_debug("MEMBER: %1");
    serialize(serializer, *data.%1);
//╚════════════════════════════════════════════════════════════════════════════════════════════════╝
            JAI, it.name);
        }
        return builder_to_string(*builder);
    }
}

// Tagged union serializer
//  Note that this only works with tagged unions that meet all of the following criteria:
//   - Tag type is not Type
//   - Every unioned member has an explicit binding
/*serialize :: (using serializer: *Serializer, data: *$T) #modify {
    TAGGED_UNION_FLAGS : Struct_Textual_Flags : .UNION | .UNION_IS_TAGGED;
    ti := T.(*Type_Info_Struct);
    if ti.type != .STRUCT then return false,
        "T is not a struct type."
    ;
    if !(ti.textual_flags & TAGGED_UNION_FLAGS) then return false,
        "T is not a tagged union."
    ;
    if ti.members[0].type.type == .TYPE then return false,
        "T's tag type is Type."
    ;
    if ti.tagged_union_bindings.count != ti.members.count-1 then return false,
        "T does not have bindings for each of its unioned members."
    ;
    return true;
} {
    log_debug("[TAGGED UNION] %", T);
    #insert -> string { builder: String_Builder;
        ti := type_info(T);
        print(*builder, #string JAI
//╔════════════════════════════════════════════════════════════════════════════════════════════════╗
    serialize(serializer, *data.%1);
    if data.%1 == {
//╚════════════════════════════════════════════════════════════════════════════════════════════════╝
        JAI, ti.members[0].name);
        for member, member_index: ti.members {
            if member_index == 0 then continue;
            binding: Type_Info_Tagged_Union_Binding;
            binding_found := false;
            for ti.tagged_union_bindings if it.member_index == member_index {
                binding, binding_found = it, true;
                break;
            }
            assert(binding_found);
            print(*builder, #string JAI
//╔════════════════════════════════════════════════════════════════════════════════════════════════╗
    case %1; serialize(serializer, *data.%2);
//╚════════════════════════════════════════════════════════════════════════════════════════════════╝
            JAI, binding.constant_value, member.name);
        }
        append(*builder, #string JAI
//╔════════════════════════════════════════════════════════════════════════════════════════════════╗
    case; //TODO: log warning?
    }
//╚════════════════════════════════════════════════════════════════════════════════════════════════╝
        JAI);
        return builder_to_string(*builder);
    }
}*/

// Tagged union serializer (NEW)
//  Only works for tagged unions that don't have Type as the type of the tag.
serialize :: (using serializer: *Serializer, data: *$T) #modify {
    TAGGED_UNION_FLAGS : Struct_Textual_Flags : .UNION | .UNION_IS_TAGGED;
    ti := T.(*Type_Info_Struct);
    if ti.type != .STRUCT then return false,
        "T is not a struct type."
    ;
    if !(ti.textual_flags & TAGGED_UNION_FLAGS) then return false,
        "T is not a tagged union."
    ;
    if ti.members[0].type.type == .TYPE then return false,
        "T's tag type is Type."
    ;
    return true;
} {
    log_debug("[TAGGED UNION] %", T);
    #insert -> string { builder: String_Builder;
        ti := type_info(T);
        print(*builder, #string JAI
//╔════════════════════════════════════════════════════════════════════════════════════════════════╗
    serialize(serializer, *data.%1);
    if data.%1 == {
//╚════════════════════════════════════════════════════════════════════════════════════════════════╝
        JAI, ti.members[0].name);
        for ti.tagged_union_bindings print(*builder, #string JAI
//╔════════════════════════════════════════════════════════════════════════════════════════════════╗
    case %1; serialize(serializer, *data.%2);
//╚════════════════════════════════════════════════════════════════════════════════════════════════╝
        JAI, it.constant_value, ti.members[it.member_index].name);
        append(*builder, #string JAI
//╔════════════════════════════════════════════════════════════════════════════════════════════════╗
    }
//╚════════════════════════════════════════════════════════════════════════════════════════════════╝
        JAI);
        return builder_to_string(*builder);
    }
}


// Zero-size serializer
serialize :: (using serializer: *Serializer, data: *$T) #modify {
    return T.(*Type_Info).runtime_size == 0;
} {
    log_debug("[ZERO-SIZE] %", T);
    // This space left intentionally blank.
}

// Bucket_Array serializer
#if #exists(Bucket_Array) then serialize :: (using serializer: *Serializer, bucket_array: *$T/Bucket_Array) {
    log_debug("[BUCKET ARRAY] %", T);
    if is_writing {
        serialize(serializer, *bucket_array.count);
        for * bucket_array.* {
            serialize(serializer, it);
        }
    } else {
        count: int;
        serialize(serializer, *count);
        for 0..count-1 {
            value: T.type;
            serialize(serializer, *value);
            bucket_array_add(bucket_array, value);
        }
    }
}

// Table serializer -- Untested!!
#if #exists(Table) then serialize :: (using serializer: *Serializer, table: *$T/Table) {
    log_debug("[TABLE] %", T);
    if is_writing {
        serialize(serializer, *table.count);
        for * table {
            serialize(serializer, *it_index);
            serialize(serializer, it);
        }
    } else {
        count: int;
        serialize(serializer, *count);
        for 0..count-1 {
            key:   T.Key_Type;
            serialize(serializer, *key);
            value: T.Value_Type;
            serialize(serializer, *value);
            table_add(table, key, value);
        }
    }
}

// Xar serializer
#if #exists(Xar) then serialize :: (using serializer: *Serializer, xar: *$T/Xar) {
    log_debug("[XAR] %", T);
    #assert false "TODO";
}

write :: (using serialize: *Serializer, data: *$T, count := 1) {
    assert(mode != .DATA, "You can't write to read-only memory, doofus!");
    log_debug("  > [%] %", count, T);

    size := size_of(T) * count;
    if !size then return;

    success := file_write(*file, data, size);
    assert(success);
}
read :: (using serializer: *Serializer, $T: Type) -> T {
    t: T;
    size, success := size_of(T), false;
    if mode == {
    case .FILE;
        success = file_read(file, *t, size);
    case .DATA;
        success = from_data.count >= size;
        if success {
            memcpy(*t, from_data.data, size);
            from_data.data  += size;
            from_data.count -= size;
        }
    }
    log_debug("  < %", t);
    assert(success);
    return t;
}
read :: (using serializer: *Serializer, $T: Type, count: int) -> [] T {
    assert(count >= 0);

    defer log_debug("  < (% %s)", count, T);

    t, size := NewArray(count, T), size_of(T)*count;

    if !size then return t;

    success := false;
    if mode == {
    case .FILE;
        success = file_read(file, t.data, size);
    case .DATA;
        success = from_data.count >= size;
        if success {
            memcpy(t.data, from_data.data, size);
            from_data.data  += size;
            from_data.count -= size;
        }
    }
    assert(success);
    return t;
}

#scope_file //==================================================================

get_common_default_module_parametric_structs :: () -> [] *Type_Info_Struct #compile_time {
    tis: [..] *Type_Info_Struct;

    #if #exists(Bucket_Array) then array_add(*tis, xx Bucket_Array);
    #if #exists(Table)        then array_add(*tis, xx Table);
    #if #exists(Xar)          then array_add(*tis, xx Xar);

    return tis;
}

#import "File";

// https://handmade.network/p/29/swedish-cubes-for-unity/blog/p/2723-how_media_molecule_does_serialization