-
-
Save stevehansen/45f0066dff69e571b249e8067e640b60 to your computer and use it in GitHub Desktop.
NDJSON File Streaming using browser Stream API + Loading Progress
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const splitOn = "\n"; | |
const newNdJsonStream = () => { | |
let buffer = ""; | |
const decoder = new TextDecoder(); | |
return new TransformStream<Uint8Array, object>({ | |
async transform(chunk, controller) { | |
chunk = await chunk; | |
// we use `stream: true` in case the chunk ends at the middle of a character | |
const string = decoder.decode(chunk, { stream: true }); | |
buffer += string; | |
const parts = buffer.split(splitOn); | |
parts | |
.slice(0, -1) | |
.forEach((part) => controller.enqueue(JSON.parse(part) as object)); | |
// Saving the last line because it may not be ended yet | |
buffer = parts[parts.length - 1]; | |
}, | |
flush(controller) { | |
if (buffer) controller.enqueue(JSON.parse(buffer) as object); | |
}, | |
}); | |
}; | |
export const fetchNdJson = (url: string) => | |
fetch(url).then((response) => ({ | |
response, | |
reader: response.body.pipeThrough(newNdJsonStream()).getReader(), | |
})); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment