Skip to content

Instantly share code, notes, and snippets.

@IanSSenne
Last active January 29, 2023 10:45
Show Gist options
  • Save IanSSenne/a5828a05e92b2b6db513571219bde3b3 to your computer and use it in GitHub Desktop.
Save IanSSenne/a5828a05e92b2b6db513571219bde3b3 to your computer and use it in GitHub Desktop.
code to transpile a limited subset of types into serializer/deserializer using bin-serde
/**
* FUNCTIONALITY EXAMPLE
*/
type PrimaryKeyA = {
$type: "a";
a: number;
};
type PrimaryKeyB = {
$type: "b";
b: string;
};
export type SuportedFeatures = {
// Objects
boolean: boolean;
booleanLiteral: true | false;
number: number;
numberLiteral: 1 | 2 | 3;
string: string;
stringLiteral: "foo" | "bar";
array: number[];
bigint: bigint;
bigintLiteral: 1n | 2n | 3n;
object: { a: number; b: string };
// Enums
enum: PieceState;
// Unions
untion: 1 | 2 | 3 | "foo" | { $type: "bar"; a: number };
optionalValue?: number;
anyValue: any;
unionWithPrimaryKey: PrimaryKeyA | PrimaryKeyB;
};
/**
* Tick Tac Toe Example
*/
type PieceState = "x" | "o" | null;
type TicTacToeBoard = PieceState[][];
type TicTacToeMove = {
x: number;
y: number;
pieces: PieceState;
};
type TicTacToeState = {
board: TicTacToeBoard;
turn: number;
winner: number;
};
export type TicTacToeMessage =
| {
$type: "move";
move: TicTacToeMove;
}
| {
$type: "state";
state: TicTacToeState;
};
import { Reader, Writer } from "bin-serde";
type PrimaryKeyA = {
$type: "a";
a: number;
};
const serializePrimaryKeyA = (
data: PrimaryKeyA,
buf: Writer = new Writer()
) => {
buf.writeFloat(data["a"]);
return buf;
};
function deserializePrimaryKeyA(buf: Reader): PrimaryKeyA {
let data;
data = {};
data["$type"] = "a";
data["a"] = buf.readFloat();
return data;
}
type PrimaryKeyB = {
$type: "b";
b: string;
};
const serializePrimaryKeyB = (
data: PrimaryKeyB,
buf: Writer = new Writer()
) => {
buf.writeString(data["b"]);
return buf;
};
function deserializePrimaryKeyB(buf: Reader): PrimaryKeyB {
let data;
data = {};
data["$type"] = "b";
data["b"] = buf.readString();
return data;
}
export type SuportedFeatures = {
// Objects
boolean: boolean;
booleanLiteral: true | false;
number: number;
numberLiteral: 1 | 2 | 3;
string: string;
stringLiteral: "foo" | "bar";
array: number[];
bigint: bigint;
bigintLiteral: 1n | 2n | 3n;
object: {
a: number;
b: string;
};
// Enums
enum: PieceState;
// Unions
untion:
| 1
| 2
| 3
| "foo"
| {
$type: "bar";
a: number;
};
optionalValue?: number;
anyValue: any;
unionWithPrimaryKey: PrimaryKeyA | PrimaryKeyB;
};
export const serializeSuportedFeatures = (
data: SuportedFeatures,
buf: Writer = new Writer()
) => {
buf.writeUInt8(data["boolean"] ? 1 : 0);
if (data["booleanLiteral"] === true) {
buf.writeUInt8(0); /*true*/
} else if (data["booleanLiteral"] === false) {
buf.writeUInt8(1); /*false*/
}
buf.writeFloat(data["number"]);
if (data["numberLiteral"] === 1) {
buf.writeUInt8(0); /*1*/
} else if (data["numberLiteral"] === 2) {
buf.writeUInt8(1); /*2*/
} else if (data["numberLiteral"] === 3) {
buf.writeUInt8(2); /*3*/
}
buf.writeString(data["string"]);
if (data["stringLiteral"] === "foo") {
buf.writeUInt8(0); /*"foo"*/ /* Literal type "foo" */
} else if (data["stringLiteral"] === "bar") {
buf.writeUInt8(1); /*"bar"*/ /* Literal type "bar" */
}
buf.writeUInt32(data["array"].length);
data["array"].forEach((item) => {
buf.writeFloat(item);
});
buf.writeUInt64(data["bigint"]);
if (data["bigintLiteral"] === 1n) {
buf.writeUInt8(0); /*1n*/ /* Literal type 1n */
} else if (data["bigintLiteral"] === 2n) {
buf.writeUInt8(1); /*2n*/ /* Literal type 2n */
} else if (data["bigintLiteral"] === 3n) {
buf.writeUInt8(2); /*3n*/ /* Literal type 3n */
}
buf.writeFloat(data["object"]["a"]);
buf.writeString(data["object"]["b"]);
serializePieceState(data["enum"], buf);
if (data["untion"] === 1) {
buf.writeUInt8(0); /*1*/
} else if (data["untion"] === 2) {
buf.writeUInt8(1); /*2*/
} else if (data["untion"] === 3) {
buf.writeUInt8(2); /*3*/
} else if (data["untion"] === "foo") {
buf.writeUInt8(3); /*"foo"*/ /* Literal type "foo" */
} else if (data["untion"]["$type"] === "bar") {
buf.writeUInt8(4); /*{ $type: "bar"; a: number }*/ /* Literal type "bar" */
buf.writeFloat(data["untion"]["a"]);
}
if (typeof data["optionalValue"] !== "undefined") {
buf.writeUInt8(1);
buf.writeFloat(data["optionalValue"]);
} else {
buf.writeUInt8(0);
}
let content_1 = JSON.stringify(data["anyValue"]);
buf.writeString(content_1);
if (data["unionWithPrimaryKey"]["$type"] === "a") {
buf.writeUInt8(0); /*PrimaryKeyA*/
serializePrimaryKeyA(data["unionWithPrimaryKey"], buf);
} else if (data["unionWithPrimaryKey"]["$type"] === "b") {
buf.writeUInt8(1); /*PrimaryKeyB*/
serializePrimaryKeyB(data["unionWithPrimaryKey"], buf);
}
return buf;
};
export function deserializeSuportedFeatures(buf: Reader): SuportedFeatures {
let data;
data = {};
data["boolean"] = !!buf.readUInt8();
let enum_1 = buf.readUInt8();
if (enum_1 === 0) {
data["booleanLiteral"] = true;
} else if (enum_1 === 1) {
data["booleanLiteral"] = false;
}
data["number"] = buf.readFloat();
let enum_2 = buf.readUInt8();
if (enum_2 === 0) {
data["numberLiteral"] = 1;
} else if (enum_2 === 1) {
data["numberLiteral"] = 2;
} else if (enum_2 === 2) {
data["numberLiteral"] = 3;
}
data["string"] = buf.readString();
let enum_3 = buf.readUInt8();
if (enum_3 === 0) {
data["stringLiteral"] = "foo";
} else if (enum_3 === 1) {
data["stringLiteral"] = "bar";
}
let len_0 = buf.readUInt32();
data["array"] = [];
for (let i = 0; i < len_0; i++) {
let value: any;
value = buf.readFloat();
data["array"].push(value);
}
data["bigint"] = buf.readUInt64();
let enum_4 = buf.readUInt8();
if (enum_4 === 0) {
data["bigintLiteral"] = 1n;
} else if (enum_4 === 1) {
data["bigintLiteral"] = 2n;
} else if (enum_4 === 2) {
data["bigintLiteral"] = 3n;
}
data["object"] = {};
data["object"]["a"] = buf.readFloat();
data["object"]["b"] = buf.readString();
data["enum"] = deserializePieceState(buf);
let enum_5 = buf.readUInt8();
if (enum_5 === 0) {
data["untion"] = 1;
} else if (enum_5 === 1) {
data["untion"] = 2;
} else if (enum_5 === 2) {
data["untion"] = 3;
} else if (enum_5 === 3) {
data["untion"] = "foo";
} else if (enum_5 === 4) {
data["untion"] = {};
data["untion"]["$type"] = "bar";
data["untion"]["a"] = buf.readFloat();
}
if (buf.readUInt8() === 1) {
data["optionalValue"] = buf.readFloat();
}
data["anyValue"] = JSON.parse(buf.readString());
let enum_6 = buf.readUInt8();
if (enum_6 === 0) {
data["unionWithPrimaryKey"] = deserializePrimaryKeyA(buf);
} else if (enum_6 === 1) {
data["unionWithPrimaryKey"] = deserializePrimaryKeyB(buf);
}
return data;
}
type PieceState = "x" | "o" | null;
const serializePieceState = (data: PieceState, buf: Writer = new Writer()) => {
if (data === "x") {
buf.writeUInt8(0); /*"x"*/ /* Literal type "x" */
} else if (data === "o") {
buf.writeUInt8(1); /*"o"*/ /* Literal type "o" */
} else if (data === null) {
buf.writeUInt8(2); /*null*/
}
return buf;
};
function deserializePieceState(buf: Reader): PieceState {
let data;
let enum_7 = buf.readUInt8();
if (enum_7 === 0) {
data = "x";
} else if (enum_7 === 1) {
data = "o";
} else if (enum_7 === 2) {
data = null;
}
return data;
}
type TicTacToeBoard = PieceState[][];
const serializeTicTacToeBoard = (
data: TicTacToeBoard,
buf: Writer = new Writer()
) => {
buf.writeUInt32(data.length);
data.forEach((item) => {
buf.writeUInt32(item.length);
item.forEach((item) => {
serializePieceState(item, buf);
});
});
return buf;
};
function deserializeTicTacToeBoard(buf: Reader): TicTacToeBoard {
let data;
let len_1 = buf.readUInt32();
data = [];
for (let i = 0; i < len_1; i++) {
let value: any;
let len_2 = buf.readUInt32();
value = [];
for (let i = 0; i < len_2; i++) {
let value: any;
value = deserializePieceState(buf);
value.push(value);
}
data.push(value);
}
return data;
}
type TicTacToeMove = {
x: number;
y: number;
pieces: PieceState;
};
const serializeTicTacToeMove = (
data: TicTacToeMove,
buf: Writer = new Writer()
) => {
buf.writeFloat(data["x"]);
buf.writeFloat(data["y"]);
serializePieceState(data["pieces"], buf);
return buf;
};
function deserializeTicTacToeMove(buf: Reader): TicTacToeMove {
let data;
data = {};
data["x"] = buf.readFloat();
data["y"] = buf.readFloat();
data["pieces"] = deserializePieceState(buf);
return data;
}
type TicTacToeState = {
board: TicTacToeBoard;
turn: number;
winner: number;
};
const serializeTicTacToeState = (
data: TicTacToeState,
buf: Writer = new Writer()
) => {
serializeTicTacToeBoard(data["board"], buf);
buf.writeFloat(data["turn"]);
buf.writeFloat(data["winner"]);
return buf;
};
function deserializeTicTacToeState(buf: Reader): TicTacToeState {
let data;
data = {};
data["board"] = deserializeTicTacToeBoard(buf);
data["turn"] = buf.readFloat();
data["winner"] = buf.readFloat();
return data;
}
export type TicTacToeMessage =
| {
$type: "move";
move: TicTacToeMove;
}
| {
$type: "state";
state: TicTacToeState;
};
export const serializeTicTacToeMessage = (
data: TicTacToeMessage,
buf: Writer = new Writer()
) => {
if (data["$type"] === "move") {
buf.writeUInt8(0); /*{
$type: "move";
move: TicTacToeMove;
}*/ /* Literal type "move" */
serializeTicTacToeMove(data["move"], buf);
} else if (data["$type"] === "state") {
buf.writeUInt8(1); /*{
$type: "state";
state: TicTacToeState;
}*/ /* Literal type "state" */
serializeTicTacToeState(data["state"], buf);
}
return buf;
};
export function deserializeTicTacToeMessage(buf: Reader): TicTacToeMessage {
let data;
let enum_8 = buf.readUInt8();
if (enum_8 === 0) {
data = {};
data["$type"] = "move";
data["move"] = deserializeTicTacToeMove(buf);
} else if (enum_8 === 1) {
data = {};
data["$type"] = "state";
data["state"] = deserializeTicTacToeState(buf);
}
return data;
}
import ts from "typescript";
import fs from "fs/promises";
let knownLiteralValues = new Set<string>([
"null",
"undefined",
"true",
"false",
]);
export const transpileFile = async (fileName: string) => {
let arrId = 0;
let contentId = 0;
let enumId = 0;
const content = await fs.readFile(fileName, "utf-8");
const sourceFile = ts.createSourceFile(
fileName,
content,
ts.ScriptTarget.ESNext,
true
);
const output: string[] = [`import { Reader, Writer } from "bin-serde";`];
let enums = new Map<string, any>();
function isKnowableLiteralValue(value: ts.Node, sourceFile: ts.SourceFile) {
return (
knownLiteralValues.has(value.getText(sourceFile)) ||
ts.isStringLiteral(value) ||
ts.isNumericLiteral(value) ||
ts.isBigIntLiteral(value) ||
enums.has(value.getText(sourceFile)) ||
!Number.isNaN(Number(value.getText(sourceFile)))
);
}
function findTypeDefinitionOrEnum(
name: string
): ts.TypeAliasDeclaration | string {
if (enums.has(name)) return enums.get(name).toString();
return findTypeDefinition(name);
}
function findTypeDefinition(name: string): ts.TypeAliasDeclaration {
const type = sourceFile.statements.find((statement) => {
if (ts.isTypeAliasDeclaration(statement)) {
return statement.name.getText(sourceFile) === name;
}
return false;
});
if (!type) throw new Error(`Type ${name} not found`);
return type as ts.TypeAliasDeclaration;
}
function getMatcher(type: ts.Node, varName: string, store: string[]) {
let matcher = "";
if (type.kind === ts.SyntaxKind.StringKeyword) {
matcher = `typeof ${varName} === "string"`;
}
if (type.kind === ts.SyntaxKind.LiteralType) {
if (type.getText(sourceFile) === "null") {
matcher = `${varName} === null`;
}
if (!Number.isNaN(Number(type.getText(sourceFile)))) {
matcher = `${varName} === ${type.getText(sourceFile)}`;
}
matcher = `${varName} === ${type.getText(sourceFile)}`;
}
if (type.kind === ts.SyntaxKind.NumberKeyword) {
matcher = `typeof ${varName} === "number"`;
}
if (type.kind === ts.SyntaxKind.BigIntKeyword) {
matcher = `typeof ${varName} === "bigint"`;
}
if (type.kind === ts.SyntaxKind.BooleanKeyword) {
matcher = `typeof ${varName} === "boolean"`;
}
if (type.kind === ts.SyntaxKind.NullKeyword) {
matcher = `${varName} === null`;
}
if (type.kind === ts.SyntaxKind.UndefinedKeyword) {
matcher = `${varName} === undefined`;
}
if (ts.isTypeReferenceNode(type)) {
let foundType = findTypeDefinitionOrEnum(type.getText(sourceFile));
if (typeof foundType === "string")
matcher = `${varName} === ${foundType}`;
else return getMatcher(foundType, varName, store);
}
if (ts.isTypeLiteralNode(type)) {
const primaryKey = type.members.find((member) => {
if (ts.isPropertySignature(member)) {
return member.name.getText(sourceFile).startsWith("$");
}
return false;
}) as ts.PropertySignature | false;
if (!primaryKey)
throw new Error(
"No primary key found, please make sure objects in a union have a property starting with '$'.\nfound: " +
type.getText(sourceFile)
);
return getMatcher(
primaryKey as ts.Node,
`${varName}[${JSON.stringify(primaryKey.name.getText(sourceFile))}]`,
store
);
}
if (ts.isPropertySignature(type)) {
if (!type.type)
throw new Error("encountered a property signature without a type");
return getMatcher(type.type, varName, store);
}
if (!matcher)
throw new Error(
`No matcher found for ${ts.SyntaxKind[type.kind]}: ` +
type.getText(sourceFile)
);
if (store.includes(matcher)) return matcher;
store.push(matcher);
}
function stringifyTypeSerializer(
type: ts.Node | undefined,
dataName = "data"
) {
if (!type) throw new Error("Type is undefined");
if (ts.isTypeLiteralNode(type)) {
return type.members
.map((member) => {
if (ts.isPropertySignature(member)) {
let preamble = member.questionToken
? `if(typeof ${dataName}[${JSON.stringify(
member.name.getText(sourceFile)
)}] !== "undefined"){buf.writeUInt8(1);`
: "";
preamble += `${stringifyTypeSerializer(
member.type,
`${dataName}[${JSON.stringify(member.name.getText(sourceFile))}]`
)};`;
preamble += member.questionToken ? `}else{buf.writeUInt8(0);}` : "";
return preamble;
} else {
throw new Error("Not a property signature");
}
})
.join("\n");
}
if (ts.isArrayTypeNode(type)) {
return [
`buf.writeUInt32(${dataName}.length);`,
`${dataName}.forEach((item) => {`,
`${stringifyTypeSerializer(type.elementType, "item")}`,
`});`,
].join("");
}
if (ts.isTypeReferenceNode(type)) {
if (enums.has(type.getText(sourceFile)))
return `buf.writeUInt32(${dataName});`;
return `serialize${type.getText(sourceFile)}(${dataName},buf)`;
}
if (type.kind === ts.SyntaxKind.StringKeyword) {
return `buf.writeString(${dataName});`;
}
if (type.kind === ts.SyntaxKind.NumberKeyword) {
return `buf.writeFloat(${dataName});`;
}
if (type.kind === ts.SyntaxKind.BooleanKeyword) {
return `buf.writeUInt8(${dataName} ? 1 : 0);`;
}
if (type.kind === ts.SyntaxKind.VoidKeyword) {
return "";
}
if (type.kind === ts.SyntaxKind.AnyKeyword) {
let varname = `content_${++contentId}`;
return [
`let ${varname} = JSON.stringify(${dataName});`,
`buf.writeString(${varname});`,
].join("");
}
if (type.kind === ts.SyntaxKind.LiteralType) {
let str = type.getText(sourceFile);
// if (str === "null") {
// return "";
// }
// if (!Number.isNaN(Number(str))) {
// return `/*${str}*/`;
// // return `buf.writeFloat(${str});`;
// }
return `/* Literal type ${str} */`;
}
if (ts.isUnionTypeNode(type)) {
let matchers: string[] = [];
let objTypes: ts.TypeLiteralNode[] = [];
type.types.forEach((t) => {
if (ts.isTypeReferenceNode(t)) {
let res = findTypeDefinitionOrEnum(t.getText(sourceFile));
if (typeof res === "string") {
let matcher = `${dataName} === ${res}`;
if (matchers.includes(res)) throw new Error("Duplicate matcher");
matchers.push(matcher);
return;
} else {
t = res.type;
}
}
getMatcher(t, dataName, matchers);
});
let intSize = 8;
while (Math.pow(2, intSize) < matchers.length) intSize *= 2;
if (intSize > 32)
throw new Error(
"Too many matchers, consider making your union type smaller"
);
return matchers
.map((matcher, i) => {
debugger;
return `if(${matcher}){buf.writeUInt8(${i});/*${type.types[i].getText(
sourceFile
)}*/${
isKnowableLiteralValue(type.types[i], sourceFile)
? ""
: stringifyTypeSerializer(type.types[i], dataName)
}}`;
})
.join("else ");
}
if (type.kind === ts.SyntaxKind.BigIntKeyword) {
return `buf.writeUInt64(${dataName});`;
}
console.log(
"SERIALIZER",
type.getText(sourceFile),
ts.SyntaxKind[type.kind]
);
process.exit(0);
}
function stringifyTypeDeserializer(
type: ts.Node | undefined,
dataName: string = "data",
isReturn: boolean = false
) {
if (type === undefined) {
throw new Error("Type is undefined");
}
if (ts.isTypeLiteralNode(type)) {
return (
`${dataName} = {};` +
type.members
.map((member) => {
if (ts.isPropertySignature(member)) {
if (member.questionToken)
return `if(buf.readUInt8() === 1){${stringifyTypeDeserializer(
member.type,
`${dataName}[${JSON.stringify(
member.name.getText(sourceFile)
)}]`
)}}`;
return stringifyTypeDeserializer(
member.type,
`${dataName}[${JSON.stringify(
member.name.getText(sourceFile)
)}]`
);
} else {
throw new Error("Not a property signature");
}
})
.join("")
);
}
if (ts.isArrayTypeNode(type)) {
let lenId = `len_${arrId}`;
arrId++;
return [
`let ${lenId} = buf.readUInt32();`,
`${dataName} = [];`,
`for(let i = 0; i < ${lenId}; i++){`,
`let value:any;${stringifyTypeDeserializer(type.elementType, "value")}`,
`${dataName}.push(value);`,
`}`,
].join("\n");
}
if (ts.isTypeReferenceNode(type)) {
if (enums.has(type.getText(sourceFile)))
return `${dataName} = buf.readUInt32();`;
return `${dataName} = deserialize${type.getText(sourceFile)}(buf)`;
}
if (type.kind === ts.SyntaxKind.StringKeyword) {
return `${dataName} = buf.readString();`;
}
if (type.kind === ts.SyntaxKind.NumberKeyword) {
return `${dataName} = buf.readFloat();`;
}
if (type.kind === ts.SyntaxKind.BooleanKeyword) {
return `${dataName} = !!buf.readUInt8();`;
}
if (type.kind === ts.SyntaxKind.VoidKeyword) {
return "";
}
if (type.kind === ts.SyntaxKind.AnyKeyword) {
return `${dataName} = JSON.parse(buf.readString());`;
}
if (type.kind === ts.SyntaxKind.LiteralType) {
let str = type.getText(sourceFile);
return `${dataName} = ${str};`;
}
if (type.kind === ts.SyntaxKind.NullKeyword) {
return `${dataName} = null;`;
}
if (ts.isUnionTypeNode(type)) {
let intSize = 8;
while (Math.pow(2, intSize) < type.types.length) intSize *= 2;
if (intSize > 32)
throw new Error(
"Too many matchers, consider making your union type smaller"
);
let id = `enum_${++enumId}`;
if (enumId >= 10) debugger;
return (
`let ${id} = buf.readUInt${intSize}();` +
type.types
.map((type, i) => {
return `if(${id} === ${i}){${
isKnowableLiteralValue(type, sourceFile)
? `${dataName} = ${type.getText(sourceFile)}`
: stringifyTypeDeserializer(type, dataName)
}}`;
})
.join("else ")
);
}
if (type.kind === ts.SyntaxKind.BigIntKeyword) {
return `${dataName} = buf.readUInt64();`;
}
console.log(
"DESERIALIZER",
type.getText(sourceFile),
ts.SyntaxKind[type.kind]
);
}
function stringifyTypeAliasDeclaration(
node: ts.TypeAliasDeclaration,
exported: boolean = false
) {
if (node.typeParameters) throw new Error("Type parameters not supported");
let serializer = `const serialize${node.name.getText(
sourceFile
)} = (data:${node.name.getText(
sourceFile
)},buf:Writer = new Writer())=>{${stringifyTypeSerializer(
node.type
)}return buf;}`;
if (exported) serializer = `export ${serializer}`;
output.push(node.getText(sourceFile));
output.push(serializer);
let deserializer = `function deserialize${node.name.getText(
sourceFile
)}(buf:Reader):${node.name.getText(
sourceFile
)}{let data;${stringifyTypeDeserializer(
node.type,
"data",
true
)};return data;}`;
if (exported) deserializer = `export ${deserializer}`;
output.push(deserializer);
}
sourceFile.statements.forEach((statement) => {
if (ts.isTypeAliasDeclaration(statement)) {
stringifyTypeAliasDeclaration(
statement,
statement.modifiers?.some(
(modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword
) ?? false
);
} else if (ts.isEnumDeclaration(statement)) {
output.push(statement.getText(sourceFile));
let name = statement.name.getText(sourceFile);
let value = new Function(
ts.transpile(statement.getText(sourceFile).replace(/^export/g, ""), {
noEmitHelpers: true,
module: ts.ModuleKind.None,
}) + `;return ${name}`
)();
Object.entries(value).forEach(([key, value]) => {
if (!Number.isNaN(+key)) return;
enums.set(`${name}.${key}`, value);
});
} else {
console.log(
"STATEMENT",
statement.getText(sourceFile),
ts.SyntaxKind[statement.kind]
);
throw new Error("Not a type alias declaration");
}
});
let serializerFile = ts.createSourceFile(
"serializer.ts",
output.join("\n"),
ts.ScriptTarget.ESNext,
true
);
let printer = ts.createPrinter({ noEmitHelpers: true });
let serializer = printer.printFile(serializerFile);
return { serializer };
};
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment