Object Graph "Package" Serialisation with clReflect

clReflectSerialise.cpp

#pragma once


#include <Core/Core.h>


namespace clutl
{
	struct Object;
	class ObjectGroup;
}


namespace game
{
	struct GameContext;


	//
	// Describes the behaviour of pointer references stored in packages
	//
	enum PackageImportType
	{
		// Points to an object in the same object group
		PIT_Local,

		// 
		PIT_Absolute,
		PIT_Parent,
	};


	//
	// A 32-bit import integer that replaces the pointer in a package file
	//
	struct PackageImport
	{
		static const i32 NB_BITS_TYPE = 2;
		static const i32 NB_BITS_LENGTH = 3;
		static const i32 NB_BITS_RESERVED = 32 - NB_BITS_LENGTH - NB_BITS_TYPE;

		// Set function for asserting if the incoming values fit within the bit-packing ranges
		inline void Set(PackageImportType t, u32 l)
		{
			core::Assert((i32)t < (1 << NB_BITS_TYPE));
			core::Assert((i32)l < (1 << NB_BITS_LENGTH));
			type = t;
			length = l;
			reserved = 0;
		}

		// What type of reference is the import?
		u32 type : NB_BITS_TYPE;

		// How many IDs are used to locate the import?
		u32 length :  NB_BITS_LENGTH;

		// Pad out the rest of the 32-bits to allow them to be set to zero
		u32 reserved : NB_BITS_RESERVED;
	};


	struct clcpp_attr(reflect) PackageContents
	{
		PackageContents()
		{
			// Package import instructions are aliased as integers so ensure they're the same size
			core::Assert(sizeof(PackageImport) == sizeof(u32));
		}

		core::Vector<u32> imports;
	};


	typedef bool (*LoadObjectPredicate)(const clcpp::Class* class_type);

	void SaveObjects(GameContext& ctx, clutl::ObjectGroup* object_group, const core::Vector<const clutl::Object*>& objects, const core::String& filename);
	bool LoadObjects(GameContext& ctx, clutl::ObjectGroup* object_group, const core::String& filename, core::Vector<clutl::Object*>& objects, LoadObjectPredicate predicate);

	// Helper wrappers that construct temporary vectors and call the above
	void SaveObject(GameContext& ctx, const clutl::Object* object, const core::String& filename);
	clutl::Object* LoadObject(GameContext& ctx, clutl::ObjectGroup* object_group, const core::String& filename);
}





#include "ObjectSerialise.h"
#include "GameContext.h"
#include "Entity.h"

#include <Core/Core.h>
#include <Core/File.h>
#include <tinycrt/tinycrt.h>

#include <clutl/Serialise.h>
#include <clutl/JSONLexer.h>
#include <clutl/FieldVisitor.h>
#include <clutl/Objects.h>


clcpp_impl_class(game::PackageContents)


namespace
{
	void ReverseInPlace(u32* values, u32 nb_values)
	{
		// Swap from either end
		for (u32 i = 0; i < nb_values / 2; i++)
		{
			u32 temp = values[i];
			values[i] = values[nb_values - i - 1];
			values[nb_values - i - 1] = temp;
		}
	}


	struct IDList
	{
		static const i32 MAX_NB_IDS = 5;
		u32 ids[MAX_NB_IDS];
		u32 nb_ids;

		IDList()
			: nb_ids(0)
		{
		}

		void Add(u32 id)
		{
			core::Assert(nb_ids < MAX_NB_IDS);
			ids[nb_ids++] = id;
		}

		void Reverse()
		{
			ReverseInPlace(ids, nb_ids);
		}

		u32 LastID() const
		{
			return ids[nb_ids - 1];
		}
	};


	void WriteObjectCreationInfo(clutl::WriteBuffer& write_buffer, const clutl::Object* object, const clutl::ObjectGroup* parent_group)
	{
		// Generate the object group path for creating this object, relative to the specified group
		IDList list;
		clutl::ObjectGroup* group = object->object_group;
		while (group != parent_group)
		{
			list.Add(group->unique_id);
			group = group->object_group;
		}

		// Write the sequence in the reverse order so it leads with the most root group
		char buffer[512];
		i32 i;
		for (i = list.nb_ids - 1; i >= 0; i--)
		{
			snprintf(buffer, sizeof(buffer), "0x%x,", list.ids[i]);
			write_buffer.WriteStr(buffer);
		}

		// Finish with the object
		snprintf(buffer, sizeof(buffer), "0x%x:\"%s\" (0x%x)",
			object->unique_id,
			object->type->name.text,
			object->type->name.hash);
		write_buffer.WriteStr(buffer);
	}


	bool ReadObjectCreationInfo(clutl::JSONContext& context, IDList& list)
	{
		// Loop parsing IDs
		while (true)
		{
			clutl::JSONToken token = LexerNextToken(context);
			if (!token.IsValid() || token.type != clutl::JSON_TOKEN_INTEGER)
				return false;
			list.Add((u32)token.val.integer);

			// Parse the only two tokens that can appear next
			token = LexerNextToken(context);
			if (token.type == clutl::JSON_TOKEN_COLON)
				break;
			if (token.type == clutl::JSON_TOKEN_COMMA)
				continue;

			// Must be invalid
			return false;
		}

		return true;
	}


	struct PointerPatch : public clutl::IFieldVisitor
	{
		PointerPatch(game::PackageContents& contents, clutl::ObjectGroup* root_group, clutl::ObjectGroup* object_group)
			: m_RootGroup(root_group)
			, m_SaveObjectGroup(object_group)
		{
			objects.clear_resize(contents.imports.size());
			memset(objects.data(), 0, objects.size() * sizeof(u32));

			// Generate the imported object list from the package contents
			for (size_t i = 1; i < objects.size(); )
			{
				game::PackageImport& import = (game::PackageImport&)contents.imports[i];

				// Saving packs IDs in reverse order
				u32* unique_ids = contents.imports.data() + i + 1;
				ReverseInPlace(unique_ids, import.length);

				// Search for the referenced object based on import type
				switch (import.type)
				{
					case game::PIT_Local:
						objects[i] = m_SaveObjectGroup->FindObjectRelative(unique_ids, import.length);
						break;
					case game::PIT_Absolute:
						objects[i] = m_RootGroup->FindObjectRelative(unique_ids, import.length);
						break;
					case game::PIT_Parent:
						objects[i] = m_SaveObjectGroup->FindObjectSearchParents(*unique_ids);
						break;
				}

				// Skip to the next import
				i += import.length + 1;
			}
		}

		void Visit(void* object, const clcpp::Type* type, const clcpp::Qualifier& qualifier) const
		{
			// Ignore non-pointers and pointers to objects that aren't named
			if (type->kind != clcpp::Primitive::KIND_CLASS)
				return;
			const clcpp::Class* class_type = (clcpp::Class*)type;
			if (!(class_type->flag_attributes & clutl::FLAG_ATTR_IS_OBJECT))
				return;

			// Alias the import description
			void** ptr = (void**)object;
			u32 ptr_id = (u32)*ptr;
			*ptr = objects[ptr_id];
		}

		core::Vector<clutl::Object*> objects;

		const clutl::ObjectGroup* m_RootGroup;
		const clutl::ObjectGroup* m_SaveObjectGroup;
	};


	struct PointerSave : public clutl::IPtrSave
	{
		PointerSave(clutl::ObjectGroup* root_group, clutl::ObjectGroup* save_object_group)
			: m_RootGroup(root_group)
			, m_SaveObjectGroup(save_object_group)
			, m_CurrentField(0)
		{
			// Take the 0-index slot in the import table so that returned ptr IDs don't look 
			// like null pointers
			m_Contents.imports.push_back(0);
		}

		bool CanSavePtr(void* ptr, const clcpp::Field* field, const clcpp::Type* type)
		{
			// Don't save raw pointers
			if (type->kind != clcpp::Primitive::KIND_CLASS)
				return false;

			// Don't save values for pointer fields that aren't derived from Object
			const clcpp::Class* class_type = type->AsClass();
			if (!(class_type->flag_attributes & clutl::FLAG_ATTR_IS_OBJECT))
				return false;

			// Only use the hash if the pointer is non-null
			if (ptr != 0)
			{
				// If the target object has no unique ID then its pointer is not meant for serialisation
				clutl::Object* object_ptr = (clutl::Object*)ptr;
				if (object_ptr->unique_id == 0)
					return false;
			}

			// Record current field for inspecting attributes
			m_CurrentField = field;

			return true;
		}

		u32 SavePtr(void* ptr)
		{
			static const u32 parent_ref_hash = core::Murmur3_HashText("parent_ref");

			// Null pointer?
			if (ptr == 0)
				return 0;

			// Ensure unnamed objects are not being filtered through
			clutl::Object* object_ptr = (clutl::Object*)ptr;
			core::Assert(object_ptr->unique_id != 0);

			// Use the object address as the key to see if this object has already been referenced
			// TODO: If two fields point to the same pointer and one field isn't marked as parent ref, it's undefined
			// whether the pointer is serialised as parent or absolute...
			u32 ptr_id = (u32)m_PointerMap.find((u32)object_ptr);
			if (ptr_id != 0)
			{
				m_CurrentField = 0;
				return ptr_id;
			}

			// Allocate a new reference in the import table
			ptr_id = m_Contents.imports.size();
			m_Contents.imports.push_back(0);
			m_PointerMap.insert((u32)object_ptr, (void*)ptr_id);

			// This is a parent reference if instructed by a field attribute
			game::PackageImport import;
			core::Assert(m_CurrentField != 0);
			if (FindPrimitive(m_CurrentField->attributes, parent_ref_hash))
			{
				import.Set(game::PIT_Parent, 1);
				m_Contents.imports.push_back(object_ptr->unique_id);
			}
			else
			{
				// Start with the reference ID
				u32 length = 1;
				m_Contents.imports.push_back(object_ptr->unique_id);

				// Walk up to the root group, recording all containing groups in reverse order
				game::PackageImportType type = game::PIT_Absolute;
				const clutl::ObjectGroup* object_group = object_ptr->object_group;
				while (object_group != m_RootGroup)
				{
					// If we encounter the save object group, this is a local reference
					if (object_group == m_SaveObjectGroup)
					{
						type = game::PIT_Local;
						break;
					}

					// Add the object group ID
					m_Contents.imports.push_back(object_group->unique_id);
					length++;

					object_group = object_group->object_group;
				}

				import.Set(type, length);
			}

			// Alias the import description as a 32-bit integer
			m_Contents.imports[ptr_id] = (u32&)import;

			// Ensure current field doesn't leak to next pointer
			m_CurrentField = 0;

			return ptr_id;
		}

		core::HashTable m_PointerMap;
		game::PackageContents m_Contents;

		const clutl::ObjectGroup* m_RootGroup;
		const clutl::ObjectGroup* m_SaveObjectGroup;

		const clcpp::Field* m_CurrentField;
	};
}


void game::SaveObjects(GameContext& ctx, clutl::ObjectGroup* object_group, const core::Vector<const clutl::Object*>& objects, const core::String& filename)
{
	// Serialise all objects to JSON
	clutl::WriteBuffer write_buffer;
	PointerSave ptr_save(ctx.root_group, object_group);
	for (size_t i = 0; i < objects.size(); i++)
	{
		const clutl::Object* object = objects[i];
		WriteObjectCreationInfo(write_buffer, object, object_group);
		clutl::SaveJSON(write_buffer, object, object->type, &ptr_save, clutl::JSONFlags::FORMAT_OUTPUT | clutl::JSONFlags::EMIT_HEX_FLOATS);
	}

	// Serialise the contents to a separate buffer so that it can be written first
	clutl::WriteBuffer contents_write_buffer;
	clutl::SaveJSON(contents_write_buffer, &ptr_save.m_Contents, clcpp::GetType<PackageContents>(), 0, clutl::JSONFlags::FORMAT_OUTPUT);

	// Write to disk
	file::File file(filename.c_str(), "w");
	if (file.IsOpen())
	{
		file.Write(contents_write_buffer.GetData(), contents_write_buffer.GetBytesWritten());
		file.Write(write_buffer.GetData(), write_buffer.GetBytesWritten());
		file.Close();
	}
}


bool game::LoadObjects(GameContext& ctx, clutl::ObjectGroup* object_group, const core::String& filename, core::Vector<clutl::Object*>& objects, LoadObjectPredicate predicate)
{
	// Open the source file
	file::File file(filename.c_str(), "r");
	if (!file.IsOpen())
		return false;

	// Read the entire file into memory
	i32 file_size = file.GetSize();
	clutl::WriteBuffer write_buffer(file_size);
	void* dest = write_buffer.Alloc(file_size);
	file.Read(dest, file_size);
	clutl::ReadBuffer read_buffer(write_buffer);

	// Read the package contents
	PackageContents contents;
	clutl::JSONError error = clutl::LoadJSON(read_buffer, &contents, clcpp::GetType<PackageContents>());
	if (error.code != clutl::JSONError::NONE)
		return false;
	
	// Parse each object in the scene file
	const clcpp::Database* reflection_db = ctx.reflection_db;
	objects.clear();
	while (read_buffer.GetBytesRemaining())
	{
		// Parse the object ID
		clutl::JSONContext context(read_buffer);
		IDList unique_id;
		if (!ReadObjectCreationInfo(context, unique_id))
			break;

		// Parse type name
		clutl::JSONToken token = LexerNextToken(context);
		if (!token.IsValid() || token.type != clutl::JSON_TOKEN_STRING)
			break;
		core::String type_name(token.val.string, token.length);

		// Skip the debug info at the end of the line
		while (read_buffer.GetBytesRemaining() && *read_buffer.ReadAt(read_buffer.GetBytesRead()) != '\n')
			read_buffer.SeekRel(1);

		// Skip the object if its type doesn't exist
		u32 type_hash = core::Murmur3_HashText(type_name);
		const clcpp::Type* type = reflection_db->GetType(type_hash);
		if (type == 0)
		{
			clutl::LoadJSON(read_buffer, 0, (clcpp::Type*)0);
			continue;
		}

		// Skip the object if the type isn't a class
		if (type->kind != clcpp::Primitive::KIND_CLASS)
		{
			clutl::LoadJSON(read_buffer, 0, (clcpp::Type*)0);
			continue;
		}
		const clcpp::Class* class_type = type->AsClass();

		// Skip the object if any available predicate says so
		if (predicate != 0 && !predicate(class_type))
		{
			clutl::LoadJSON(read_buffer, 0, (clcpp::Type*)0);
			continue;
		}

		// Skip the object data if the object already exists
		clutl::Object* object = object_group->FindObjectRelative(unique_id.ids, unique_id.nb_ids);
		if (object != 0)
		{
			clutl::LoadJSON(read_buffer, 0, (clcpp::Type*)0);
		}
		else
		{
			// Locate the group the object wants to be created in
			clutl::ObjectGroup* creation_group = object_group;
			if (unique_id.nb_ids > 1)
				creation_group = (clutl::ObjectGroup*)object_group->FindObjectRelative(unique_id.ids, unique_id.nb_ids - 1);
			if (creation_group == 0 || creation_group->type->kind != clcpp::Primitive::KIND_CLASS)
				break;
			if (!(creation_group->type->AsClass()->flag_attributes & clutl::FLAG_ATTR_IS_OBJECT_GROUP))
				break;

			// Create the object, parse it and keep a collection of loaded objects
			object = clutl::CreateObject(type, unique_id.LastID(), creation_group);
			clutl::JSONError error = clutl::LoadJSON(read_buffer, object, type);
			if (error.code == clutl::JSONError::NONE)
				objects.push_back(object);
		}
	}

	// Patch up all loaded pointers
	// NOTE: if any object groups are activated, the last one most be the lowest in the tree
	PointerPatch patch(contents, ctx.root_group, object_group);
	for (size_t i = 0; i < objects.size(); i++)
	{
		clutl::Object* object = objects[i];
		clutl::VisitFields(object, object->type, patch, clutl::VFT_Pointers);
	}

	// Execute PreLoad and PostLoad as distinct steps
	// TODO: Custom flags for DerivesFrom replacement!!!
	for (size_t i = 0; i < objects.size(); i++)
	{
		clutl::Object* object = objects[i];
		if (object->type->DerivesFrom(clcpp::GetTypeNameHash<game::Component>()))
			((game::Component*)object)->OnPreLoad(ctx);
	}
	for (size_t i = 0; i < objects.size(); i++)
	{
		clutl::Object* object = objects[i];
		if (object->type->DerivesFrom(clcpp::GetTypeNameHash<game::Component>()))
			((game::Component*)object)->OnPostLoad(ctx);
	}

	return true;
}


void game::SaveObject(GameContext& ctx, const clutl::Object* object, const core::String& filename)
{
	core::Assert(object != 0);
	core::Vector<const clutl::Object*> objects;
	objects.push_back(object);
	SaveObjects(ctx, object->object_group, objects, filename);
}


clutl::Object* game::LoadObject(GameContext& ctx, clutl::ObjectGroup* object_group, const core::String& filename)
{
	core::Vector<clutl::Object*> objects;
	LoadObjects(ctx, object_group, filename, objects, 0);

	// Keep removing any loaded objects until only one is left
	// TODO: This is an awful way of doing this - should not not the objects in the first place!
	while (objects.size() > 1)
	{
		Delete(objects[1]);
		objects.remove_unstable(1);
	}

	if (objects.size())
		return objects[0];
	
	return 0;
}