Last active
October 24, 2023 16:58
-
-
Save nhrones/a872b581a4685c2b6374b080d318123d to your computer and use it in GitHub Desktop.
Expandable Codec Buffer
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/** A resizable buffer */ | |
export let accumulator = new Uint8Array( 1 << 14 ) // 16384 | |
/** CodecBuffer class */ | |
export class CodecBuffer { | |
/** The next available byte (tail-pointer) */ | |
nextByte = 0 | |
/** extract the encoded bytes | |
* @returns - a trimmed encoded buffer | |
*/ | |
extractEncoded() { | |
return accumulator.slice(0, this.nextByte) | |
} | |
/** check fit - expand accumulator as required */ | |
requires(bytesRequired: number) { | |
if (accumulator.length < this.nextByte + bytesRequired) { | |
let newAmt = accumulator.length | |
while (newAmt < this.nextByte + bytesRequired) newAmt *= 2 | |
const newStorage = new Uint8Array(newAmt) | |
newStorage.set(accumulator, 0) | |
accumulator = newStorage | |
console.log('Increased accumulator capacity to - ', accumulator.byteLength) | |
} | |
} | |
/** add a byte to the accumulator */ | |
appendByte(val: number) { | |
this.requires(1) | |
accumulator[this.nextByte++] = val | |
} | |
/** add a buffer to the accumulator */ | |
appendBuffer(buf: Uint8Array) { | |
const len = buf.byteLength | |
this.requires(len) | |
accumulator.set(buf, this.nextByte) | |
this.nextByte += len | |
} | |
} |
The complete multi-part key encoder
import {
BYTES,
DOUBLE,
FALSE,
TRUE,
NULL,
STRING,
} from './types.ts'
//===========================================
// Encode a multipart key to a byte array
//===========================================
/** Internal function - see pack() below */
const encodeKey = (accumulator: CodecBuffer, item: KeyPart) => {
if (item === undefined) throw new TypeError('Packed element cannot be undefined')
else if (item === null) accumulator.appendByte(NULL)
else if (item === false) accumulator.appendByte(FALSE)
else if (item === true) accumulator.appendByte(TRUE)
else if (item.constructor === Uint8Array || typeof item === 'string') {
let itemBuf
if (typeof item === 'string') {
itemBuf = new TextEncoder().encode(item)
accumulator.appendByte(STRING)
}
else {
itemBuf = item
accumulator.appendByte(BYTES)
}
for (let i = 0; i < itemBuf.length; i++) {
const val = itemBuf[i]
accumulator.appendByte(val)
if (val === 0)
accumulator.appendByte(0xff)
}
accumulator.appendByte(0)
} else if (Array.isArray(item)) {
// Embedded child tuple.
throw new Error('Nested Tuples are not supported!')
} else if (typeof item === 'number') {
// Encode as a double precision float.
accumulator.appendByte(DOUBLE)
accumulator.appendBuffer(encodeDouble(item))
} else if (isBigInt(item)) {
// Encode as a BigInt
encodeBigInt(accumulator, item as bigint)
} else {
throw new TypeError('Item was not a valid FDB keyPart type!')
}
}
/** Internal function - see pack() below */
function packRawKey(part: KeyPart[]): Uint8Array {
if (part === undefined
|| Array.isArray(part)
&& part.length === 0) return new Uint8Array(0)
if (!Array.isArray(part)) {
throw new TypeError('pack must be called with an array')
}
const accumulator= new CodecBuffer()
for (let i = 0; i < part.length; i++) {
encodeKey(accumulator, part[i])
}
// finally, when all parts have been encoded, we extract and return our FDB-encoded key-blob
return accumulator.extractEncoded()
}
/**
* Encode the specified item or array of items into a buffer.
* pack() and unpack() are the main entry points
*/
export function pack(parts: KeyPart[]): Uint8Array {
const packedKey = packRawKey(parts)
return packedKey
}
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Unlike buffer-concat(), a buffer create/swap is only required when an expansion is required!
Initial buffer size could be large enough to preclude most expansions.
In line 32 and line 39 we check that we have capacity -> requires(number)
The requires method (line 18) checks capacity, and expands the buffer only as required.
The extractEncoded method (line 13) extracts the exact encoded contents from the expandable buffer.
Note:
The Uint8Array.set() methods (lines 23 and 40), are very performant and efficient!
TypedArrays are heavily optimized in V8!
The underlying ArrayBuffer represents a contiguous block of binary data.
Because of this, the set operation is extremely fast.
Please see:
https://forum.babylonjs.com/t/optimizing-performance-of-binarywriter-resizebuffer/37516
Usage -- encoding a multi-part Denokv-key