Jai implementation of Media Molecule's LBP serialization system

Serialization.jai

/*

How to use this:


 0. Define your data, and #load (*NOT* #import) this file.

        My_Data :: struct {
            lives    := 3;
            radians: float;
        }
        global_my_data: My_Data;

        #load "Serialization.jai";


 1. Define your Serializer with your data type and a version enum as parameters.

        My_Serializer :: Serializer(
            DATA    = My_Data,
            VERSION = enum {
                INITIAL;
            }
        );


 2. Write a serialize(using *My_Serializer, *My_Data) procedure, which invokes
    ADD(v, ..xyz), where v is the initial version from the enum, and xyz
    is one string for each member name.

        serialize :: (using serializer: *My_Serializer, data: *My_Data) {
            ADD(.INITIAL, "lives", "radians");
        }


 3. Use Serialize()/Deserialize() when you want to save/load your data. Both
    take your My_Serializer type as the first argument, a filename as the second
    argument, and a pointer to the My_Data that you want to (de)serialize.

        if ctrl_s_pressed()
            then Serialize(My_Serializer, "data.dat", *global_my_data);

        if ctrl_l_pressed()
            then Deserialize(My_Serializer, "data.dat", *global_my_data);


 4. When you want to add a member to My_Data, first add a new version to
    My_Serializer's VERSION enum, then add an ADD(v, m) line to your serialize()
    procedure.

        My_Data :: struct {
            lives      := 3;
            radians:   float;
            continues: int;
        }

        My_Serializer :: Serializer(
            DATA    = My_Data,
            VERSION = enum {
                INITIAL;
                ADDED_CONTINUES;
            }
        );

        serialize :: (using serializer: *My_Serializer, data: *My_Data) {
            ADD(.INITIAL, "lives", "radians");
            ADD(.ADDED_CONTINUES, "continues");
        }


 5. When you want to remove a member from My_Data, once again, add a new version
    to My_Serializer's VERSION enum, then find the ADD() line where the member
    was added, make note of the version it was added, and add a REM(v0, v1, m)
    line to your serialize() procedure, where v0 is the version it was added,
    and v1 is the version it was removed. Then, edit that ADD() line to remove
    the string for that member.

        My_Data :: struct {
            lives      := 3;
            continues: int;
        }

        My_Serializer :: Serializer(
            DATA    = My_Data,
            VERSION = enum {
                INITIAL;
                ADDED_CONTINUES;
                REMOVED_RADIANS;
            }
        );

        serialize :: (using serializer: *My_Serializer, data: *My_Data) {
            ADD(.INITIAL, "lives");
            ADD(.ADDED_CONTINUES, "continues");
            REM(.INITIAL, .REMOVED_RADIANS, "radians");
        }


 6. If you want My_Data to contain any other data structures you've written that
    may change in the future, write a serialize(*My_Serializer, *T) procedure
    for it as well, just like you did for My_Data:

        My_Data :: struct {
            lives      := 3;
            continues: int;
            other_data: Other_Data;
        }

        Other_Data :: struct {
            chosen_deity_name: string;
            blunders:          int;
        }

        My_Serializer :: Serializer(
            DATA    = My_Data,
            VERSION = enum {
                INITIAL;
                ADDED_CONTINUES;
                REMOVED_RADIANS;
                ADDED_OTHER_DATA;
            }
        );

        serialize :: (using serializer: *My_Serializer, data: *My_Data) {
            ADD(.INITIAL, "lives");
            ADD(.ADDED_CONTINUES, "continues");
            REM(.INITIAL, .REMOVED_RADIANS, "radians");
            ADD(.ADDED_OTHER_DATA, "other_data");
        }

        serialize :: (using serializer: *My_Serializer, data: *Other_Data) {
            ADD(.ADDED_OTHER_DATA, "chosen_deity_name", "blunders");
        }


 7. If you want to add a member to any of your data structures that you don't
    want to be serialized/deserialized, put a @NoSerialize note on it. (The note
    text can be customized by setting NO_SERIALIZE_NOTE in the Serializer if you
    so desire.)

        My_Data :: struct {
            lives      := 3;
            continues: int;
            other_data: Other_Data;
            some_table_that_we_fill_at_runtime: Table(int, string); @NoSerialize
        }

*/

Serializer :: struct(
    DEBUG                     := false,
    DATA:                Type,                  // your data
    VERSION:             Type,                  // an enum
    NO_SERIALIZE_NOTE         := "NoSerialize",
    GENERIC_STRUCT_BLACKLIST: ..Type            // any polymorphic Types you don't want auto-serialized
) {
    #assert(type_info(VERSION).type == .ENUM && type_info(VERSION).values.count > 0);
    DATA_VERSION_LATEST :: #run -> VERSION {
        ti := type_info(VERSION);
        return xx ti.values[ti.values.count-1];
    };

    data_version: VERSION;
    file:         File;
    is_writing:   bool;

    ADD :: (data_version: VERSION, $field_names: ..string) #expand {
        if `serializer.data_version >= data_version {
            #insert -> string { builder: String_Builder;
                for field_names print(*builder, #string JAI
//╔════════════════════════════════════════════════════════════════════════════════════════════════╗
            `serialize(`serializer, *`data.%);
//╚════════════════════════════════════════════════════════════════════════════════════════════════╝
                JAI, it);
                return builder_to_string(*builder);
            }
        }
    }

    REM :: (data_version_added: VERSION, data_version_removed: VERSION, $T: Type, field_name: string) {
        #insert #run tprint(#string JAI
//╔════════════════════════════════════════════════════════════════════════════════════════════════╗
        `%1: %2;
        if
            `serializer.data_version >= data_version_added   &&
            `serializer.data_version <  data_version_removed
        then `serialize(`serializer, *`%1);
//╚════════════════════════════════════════════════════════════════════════════════════════════════╝
        JAI, field_name, T);
    }

    #if DEBUG
        then log_debug :: (message: string, args: ..Any) { log(message, args); }
        else log_debug :: (#discard message: string, #discard args: ..Any) {}
}

// Call this to serialize your data
Serialize :: ($SERIALIZER: Type, filename: string, data: *SERIALIZER.DATA) {
    using serializer := SERIALIZER.{
        data_version = SERIALIZER.DATA_VERSION_LATEST,
        is_writing   = true
    };
    log_debug("=== SERIALIZING ===");
    file=, file_open_success := file_open(filename, for_writing=true);
    assert(file_open_success);
    serialize(*serializer, *serializer.data_version);
    serialize(*serializer, data);
    file_close(*file);
    log_debug("=== DONE ===");
}

// Call this to deserialize your data
Deserialize :: ($SERIALIZER: Type, filename: string, data: *SERIALIZER.DATA) -> bool {
    using serializer := SERIALIZER.{
        is_writing   = false
    };
    log_debug("=== DESERIALIZING ===");
    file=, file_open_success := file_open(filename, for_writing=false);
    assert(file_open_success);
    serialize(*serializer, *serializer.data_version);
    serialize(*serializer, data);
    file_close(*file);
    log_debug("=== DONE ===");
    return true;
}

//--------------- Everything below is internal stuff for the serialization system ------------------

// Array view serializer
serialize :: (using serializer: *Serializer, data: *[] $T) {
    if is_writing {
        write(serializer, *data.count);
        write(serializer, data.data, data.count);
    } else {
        count := read(serializer, int);
        data.* = read(serializer, T, count);
    }
}

// string serializer
serialize :: (using serializer: *Serializer, data: *string) {
    serialize(serializer, data.(*[] u8));
}

// Scalar serializer
serialize :: (using serializer: *Serializer, data: *$T) #modify {
    ti := T.(*Type_Info_Variant);
    tt := ti.type;
    while true if tt == {
        case .BOOL;    #through;
        case .INTEGER; #through;
        case .ENUM;    #through;
        case .FLOAT;   return true;
        case .VARIANT; tt = ti.variant_of.type;
        case; return false;
    }
    return false;
} {
    log_debug("[GENERIC SCALAR] %", T);
    if is_writing
        then write(serializer, data);
        else data.* = read(serializer, T);
}

// Generic struct serializer
serialize :: (using serializer: *$SERIALIZER/Serializer, data: *$T) #modify {
    ti := T.(*Type_Info_Struct);
    if ti.type != .STRUCT then return false;

    tis := SERIALIZER.(*Type_Info_Struct);
    for member: tis.specified_parameters if member.name == "GENERIC_STRUCT_BLACKLIST" {
        for 0..member.type.(*Type_Info_Array).array_count-1
            if ti == xx (tis.constant_storage.data + member.offset_into_constant_storage + size_of(Type)*it).(*Type).*
                then return false;
        break;
    }

    if !ti.polymorph_source_struct then return true;
    for get_generic_struct_serializer_polymorph_source_blacklist()
        if ti.polymorph_source_struct == it
            then return false;

    return true;
} {
    log_debug("[GENERIC STRUCT] %", T);
    #insert -> string { builder: String_Builder;
        for type_info(T).members {
            if (it.flags & .CONSTANT) || (it.flags & .OVERLAY) then continue;
            no_serialize := false;
            for it.notes if it == NO_SERIALIZE_NOTE { no_serialize = true; break; }
            if no_serialize then continue;
            print(*builder, #string JAI
//╔════════════════════════════════════════════════════════════════════════════════════════════════╗
    serialize(serializer, *data.%);
//╚════════════════════════════════════════════════════════════════════════════════════════════════╝
            JAI, it.name);
        }
        return builder_to_string(*builder);
    }
}

// Table serializer
#if #exists(Table) then serialize :: (using serializer: *Serializer, table: *$T/Table) {
    log_debug("[TABLE] %", T);
    if is_writing {
        serialize(serializer, *table.count);
        for * table {
            serialize(serializer, *it_index);
            serialize(serializer, it);
        }
    } else {
        count: int;
        serialize(serializer, *count);
        for 0..count-1 {
            key:   T.Key_Type;
            serialize(serializer, *key);
            value: T.Value_Type;
            serialize(serializer, *value);
            table_add(table, key, value);
        }
    }
}

write :: (using serialize: *Serializer, data: *$T, count := 1) {
    log_debug("  > [%] %", count, T);
    success := file_write(*file, data, size_of(T) * count);
    assert(success);
}
read :: (using serializer: *Serializer, $T: Type) -> T {
    t: T;
    success := file_read(file, *t, size_of(T));
    log_debug("  < %", t);
    assert(success);
    return t;
}
read :: (using serializer: *Serializer, $T: Type, count: int) -> [] T {
    t := NewArray(count, T);
    success := file_read(file, t.data, size_of(T) * count);
    assert(success);
    return t;
}

get_generic_struct_serializer_polymorph_source_blacklist :: () -> [] *Type_Info_Struct #compile_time {
    tis: [..] *Type_Info_Struct;

    #if #exists(Table) then array_add(*tis, xx Table);

    return tis;
}

// https://handmade.network/p/29/swedish-cubes-for-unity/blog/p/2723-how_media_molecule_does_serialization