[highland.js ETL] #etl #highland #tools

consts.ts

export const Kbyte = 1024;
export const Mbyte = Kbyte * Kbyte;
export const Gbyte = Kbyte * Mbyte;

fromUrl.ts

import { fetch } from "undici";
import _H from "highland";
import { promiseToStream } from "@wmakeev/highland-tools";
import { Readable } from "node:stream";

export function fromUrl(url: string) {
  return _H([url])
    .map(async (url) => {
      const resp = await fetch(url);

      if (resp.body == null) {
        throw new Error("Empty response body");
      }

      return Readable.fromWeb(resp.body);
    })
    .map(promiseToStream)
    .sequence()
    .flatMap((body) => {
      return _H(body);
    });
}