protocol.cpp

// Defines the dynamic protocol that everything uses to communicate with each other.
// This encodes a bunch of keys, implicit integer ids (just indices of the keys), etc., as a lookup table
// that is transmitted when clients first establish contact.
// Of note:
//	- each client has their own, independent, and dynamically-defined protocol / packet translation table.
//	  If client A is communicating with client/server B, A -> B does not necessarily use the same protocol as B -> A.
// 	- This *does* mean that each client has to retain/store a lookup table for *every other* client/server
//    that they're connected to for inbound communications. Servers/clients use the same table for all of 
//    their outbound communications (which they broadcast to everyone else).
//  - The *advantage* of this approach is that A and B can be running different versions without being
//    binary-incompatabile. A could have some extra entity properties that B doesn't, for example, and
// 	  B would just gracefully ignore the properties that it didn't understand (which is what we currently have
//	  (sort-of), but this is slightly better -- A could also REMOVE some properties (or reorder them, or something),
//	  which would totally break our current enum-based approach).
//  - Fields are still defined in terms of fixed binary encodings of types (and major changes to these
//	  would break binary compatability), but the properties themselves can be defined/redefined in terms
//	  already *existing* types without breaking anything.
struct CommProtocolDefn {
	struct Field {
		Field (const std::string &key, DataEncoding type, uint16_t fields)
			: key(key), type(type), fieldLength(fieldLength) {}
		std::string 	key;
		DataEncoding  	type;
		uint16_t 		fieldLength;
	};
	std::vector<Field> fields;

public:
	CommProtocolDefn () {}
	CommProtocolDefn (std::vector<Field> & field) :
		fields(std::move(field)) {}

	void addKeyType (const std::string &key, DataEncoding type, uint16_t fieldLength) {
		fields.emplace_back(key, type, fieldLength);
	}
};

// Packet struct.
// Encodes CommProtocolDefn into a singly-allocated chunk of memory that can be memcpied, etc,
// without pointers or dangling references. String data is written to a section at the end, and
// references are implemented as uint16_t indices / pseudo-pointers.
//
// Do NOT to allocate this normally (you'll get back a 32-bit data structure with no fields, lol),
// since its structure is heavily dependent on a) an array of Field(s), and b) an array of string data
// being allocated and populated as part of the CommProtocolPacket allocation. This is so we can use
// memcpy, etc., and eliminate pointers (which can't be sent across the wire ofc).
// use makePacket() to create a packet, and fromPacket() to convert it back into a CommProtocolDefn.
struct CommProtocolPacket {
	uint16_t packet_size;
	uint16_t num_keys;

	struct Field {
		uint16_t key;	// char * ptr as offset from the start of this struct
		uint16_t type;	// DataEncoding (as uint16_t)
		uint16_t fieldLength;
		// uint16_t id;	// the id is implicit
	};
};


// Creates a packet from the provded CommProtocolDefn. The entire data structure is allocated from one chunk of memory,
// so it is memcpy-able. Strings and the field array are allocated from the same memory block as the struct.
CommProtocolPacket* makePacket (const CommProtocolDefn & p) {
	assert((header == nullptr) == (headerSize == 0));

	const size_t structSize = sizeof(CommProtocolPacket);
	const size_t fieldSize  = sizeof(CommProtocolPacket::Field);
	const size_t fieldOffset = fieldSize * p.fields.size();
	size_t 		 numStringBytes = 0;
	
	// Calculate memory needed for all key strings 
	for (const auto & field : p.fields) {
		numStringBytes += field.key.size() + 1;
	}

	void* packetChunk = malloc(structSize + fieldOffset + numStringBytes);

	if (header != nullptr)
		std::copy(static_cast<ubyte*>(header), static_cast<ubyte*>(header + headerSize), static_cast<ubyte*>(packetChunk));
	
	auto packet = static_cast<CommProtocolPacket*>(packetChunk + headerSize);
	packet->packet_size = structSize + numStringBytes + headerSize;
	packet->num_keys = p.size();
	char * str_mem = static_cast<char*>(packetChunk + headerSize + structSize);


	Field* p_field = static_cast<Field*>(packetChunk + structSize);
	char * p_str   = static_cast<char*> (packetChunk + structSize + fieldOffset);

	for (uint16_t i = 0, k = 0; i < p.fields.size(); ++i) {
		const char * str = p.fields[i].key.c_str();
		size_t 	     len = p.fields[i].key.size();

		p_field[i].key         = k;
		p_field[i].type        = p.fields[i].type;
		p_field[i].fieldLength = p.fields[i].fieldLength;

		std::copy(str, str + len, p_str + k * sizeof(char));
		p_str[k + len + 1] = '\0';
		k += len + 1;
	}

	return packetChunk;
}

CommProtocolDefn fromPacket (const CommProtocolPacket * packet) {
	assert(packet != nullptr);
	assert(packet->packet_size > sizeof(CommProtocolPacket) + 
		sizeof(CommProtocolPacket::Field) * packet->num_keys)

	std::vector<CommProtocolDefn::Field> fields (packet->num_keys);

	auto packet_fields = static_cast<CommProtocolPacket::Field*>(packet + sizeof(CommProtocolPacket));
	auto packet_str    = static_cast<char*>(packet_fields + sizeof(CommProtocolPacket::Field) * packet->num_keys);

	for (uint16_t i = 0; i < packet->num_keys; ++i) {
		Field * field = &(packet_fields[i]);
		const char * key_str = packet_str + field[i].key;
		fields.emplace_back({ key_str }, (DataEncoding)field->type, field->fieldLength);
	}

	return CommProtocolDefn { std::move(fields) };
}





template <typename Class>
struct NetworkPropertyBinding {
	NetworkPropertyBinding (std::string packetKey, FieldEncoding fieldEncoding, 
		std::function<void(Class&, BitstreamReader&)> reader) :

	std::string packetKey;
	FieldEncoding fieldEncoding;
	std::function<void(Class&, BitStreamReader&)> reader;
};

template <typename Class>
struct NetworkPropertyBindings {
	NetworkPropertyBinding (const std::initializer_list<NetworkPropertyBinding<Class>> & elems) {
		for (auto binding : elems) {
			bindings.emplace(binding.packetKey, binding);
			// registeredReaders.emplace(elems.packetKey, elems.reader);
			// fieldEncodings.emplace(elems.packetKey, elems.fieldEncoding);
		}
	}

	std::map<std::string, NetworkPropertyBinding> bindings;

	std::map<std::string, std::function<void(Class&, BitstreamReader&)>> 	registeredReaders;
	std::map<std::string, FieldEncoding>									fieldEncodings;

	bool handlePacketKey (Class& cls, CommReader &reader) {
		auto key = reader.getKey();
		if (bindings.contains(key)) {
			bindings[key].reader(cls, reader.fieldBitstream());
			return true;
		}
		return false;
	}

	void registerInterface (CommProtocolDefn &defn) {
		for (auto const & binding : bindings) {
			defn.add(binding.packetKey, binding.fieldEncoding);
		}
	}
};


class EntityNetworkInterface {
	SomeContainerOrInterface<Entity> _entities;
public:
	// Refers to the currently active entity (from a network perspective).
	// This value is never null; instead, we have a dummy "null" value defined in memory, so that packet commands
	// are always defined.
	// For example, suppose someone tries to write some packets to an entity that either doesn't exist, or has been
	// destroyed and they don't yet know about it. Instead of adding extra checks and branches to handle this scenario,
	// we just create an extra, unused entity and forward all bad requests to that.
	Entity * _activeEntity;


	static Entity * nullEntity () {
		static Entity emptyEntity {};
		return &emptyEntity;
	}

	void updateEntities () {
		PacketList packetList;

		for (auto entity : _entities {
			// Send the entities' key (essentially starting a packet transmission for this entity).
			packetList.write("Entity.METAKEY", entity.id(), isMetaKey=true);

			// Write any properties that have changed since the last update
			for (auto kv : entity.properties) {
				auto key   = kv.first;
				auto value = kv.second;
				if (value.flags.get(NEEDS_NETWORK_UPDATE)) {
					packetList.write(key, value);
					value.flags.clear(NEEDS_NETWORK_UPDATE);
				}
			}
		}

		// Dispatches our sent commands via one or more packets.
		// PacketList is a high-level abstraction for sending entity properties, etc., as under the hood, it:
		//  - prefixes each packet with a timestamp (uniform across all packets sent by this command)
		//	- may add additional metakeys.
		// (if a packet stream needs to be split into two or more packets, it'll write the last used metakey to
		//  the start of each packet so that the sent data is still valid even if the packets arrive out of order.
		//  A metakey is just a regular key with extra semantic meaning (eg. declaring which entity our property
		//  updates should affect), and it's defined using PacketList::write(..., isMetaKey=true)).
		network.sendPackets(packetList);
	}

	// Keys, encoding patterns / types, and read functions are defined here.
	// Read functions (eg. readVec3) are defined generically, since
	//	- we define the write encoding 			(ie. we're transmitting Entity.position using VEC3_FULL)
	//	- we do NOT define the read encoding (we know it's a glm::vec3, but how it's actually encoded and sent over
	//	  the wire is totally up to the client / server on the other end, and this could potentially change between
	// 	  releases.
	//	- Thus we use readVec3(reader), and NOT readVec3Full(reader), even though we're transmitting as VEC3_FULL.
	//
	static NetworkPropertyBindings<EntityNetworkInterface>
	bindings {
		{ "Entity.METAKEY", FIELD_ENCODING_QUUID, [](EntityNetworkInterface& this, BitStreamReader& reader) {
			auto id = readQuuid(reader);
			this._activeEntity = this._entities.contains(id) ? this._entities.get(id) : this._entities.nullEntity();
		}},
		{ "Entity.position", FIELD_ENCODING_VEC3_FULL, [](EntityNetworkInterface& this, BitStreamReader& reader) {
			this._activeEntity->setProperty("position", readVec3(reader));
		}},
		{ "Entity.rotation", FIELD_ENCODING_QUAT_OPTIMIZED, [](EntityNetworkInterface& this, BitStreamReader &reader) {
			this._activeEntity->setProperty("rotation", readQuat(reader));
		}},
	}

	// Tries to read a network packet. Returns true if we read anything, and false otherwise.
	// Determining whether this is an error or not (and how to report it) is up to the caller.
	bool readNetworkPacket (CommReader &reader) {
		bool read = false;
		while (bindings.handlePacketKey(*this, reader) && !reader.atEnd()) {
			reader.advanceKey();
			read = true;
		}
		return read;
	}
};

class NetworkInterface {
	EntityNetworkInterface _entityInterface;
	FooNetworkInterface    _fooInterface;

public:
	void handleNetworkPacket(const Packet &packet) {
		auto reader = makeReader(packet);
		while (!reader.atEnd()) {
			if (_entityInterface.readNetworkPacket(reader)) {}
			else if (_fooInterface.readNetworkPacket(reader)) {}
			// ...
			else {
				logUnhandledPacketKey(reader.getKey());
				reader.advanceKey();
			}
		}
	}
};

/// Wraps access to a network packet in a bistream reader.
/// This is *tightly* integrated with the CommReader impl
class BitStreamReader {
	// Data stream
	uint64_t * _dataStream;
	size_t     _bitOffset;

	uint64_t _buffer[32];

	// Field info set by the packet I/O interface
	FieldEncoding _fieldEncoding;
	size_t 		  _expectedFieldBits;
public:
	FieldEncoding fieldEncoding () const { return _fieldEncoding; }

};

// Sample read impl (we use a switch statement to dispatch to the appropriate endoder/decoder function).
glm::vec3 readVec3 (BitStreamReader &reader) {
	switch (reader.fieldEncoding()) {
		case FIELD_ENCODING_VEC3_FULL: return readVec3Full(reader);
		default: throw UnsupportedEncodingError { encoding };
	}
}

glm::vec3 readVec3Full (BitStreamReader &reader) {
	assert(sizeof(float) == 4);
	float * values = reader.readBits(sizeof(float) * 8 * 3);
	return glm::vec3 { values[0], values[1], values[2] };
}


// This is not currently used, but would presubaly be called by PacketList::write(key, value) somehow.
void writeVec3 (BitStreamWriter &writer, const glm::vec3 &v, DataEncoding encoding) {
	switch (encoding) {
		case FIELD_ENCODING_VEC3_FULL: writeVec3Full(writer, v); break;
		default: throw UnsupportedEncodingError { encoding };
	}
}
void writeVec3Full (BitStreamWriter &writer, const glm::vec3 &v) {
	assert(sizeof(float) == 4);

	writer.writeBits(sizeof(float) * 8, &v.x);
	writer.writeBits(sizeof(float) * 8, &v.y);
	writer.writeBits(sizeof(float) * 8, &v.z);
}



// CommReader is NOT defined here, but it's basically just a struct that wraps a (recieved) packet and
// translator (CommProtocolDefn), and sequentially iterates over key/value pairs (where keys are varint-sized
// ids mapped back to string keys, and the value / field(s) are just a bundle of bits whose size, meaning,
// and decoding process are defined by the communication protocol).

// Likewise, PacketList would have a pretty straightforward implementation.
// (accumulate a list of micro-packets (key/value pairs, with a few keys marked as metakeys), and batch them
// together into full packets (multiple packets if necessary), with headers (or whatever), and a timestamp of
// when the packet list was sent. Metakeys would also be sent again, with the last one used at the start of
// every additional packet, as mentioned earlier).