A standalone decoder for earth.nullschool.net .epak files

epak-test.js

import { decodeEpak } from "./epak.mjs";

const buf = await fetch(
  "https://gaia.nullschool.net/data/gfs/current/current-wind-isobaric-1000hPa-gfs-0.5.epak",
).then((r) => r.arrayBuffer());

const { header, blocks } = decodeEpak(buf);
const u = blocks[header.variables["u-component_of_wind_isobaric"].data.block];
const v = blocks[header.variables["v-component_of_wind_isobaric"].data.block];

// each Float32Array(720*361), index = row*720 + col, row 0 = lat +90, col 0 = lon 0°E, 0.5° steps

epak.mjs

// epak.mjs — standalone decoder for earth.nullschool.net .epak files.
//
// Own implementation of the format, written against the behaviour of the
// site's codec (src/codec/decoder.js, recovered from its source map).
// Container: "head" i32be json | ( type4 i32be payload )* | "tail"
// ppak payload: i32be cols, i32be rows, i32be grids, f32be exp, varpack bytes
// Decode pipeline: varpack ints -> undo 2D plane deltas -> divide by 10^exp
// Works in Node and the browser (pass an ArrayBuffer).

export function decodeEpak(buffer, { headerOnly = false } = {}) {
  const view = new DataView(buffer);
  const td = new TextDecoder();
  let off = 0;

  const tag4 = () => td.decode(new Uint8Array(buffer, off, 4));

  if (tag4() !== "head") throw new Error("not an epak file (missing 'head')");
  off += 4;
  const headLen = view.getInt32(off); off += 4;
  const header = JSON.parse(td.decode(new Uint8Array(buffer, off, headLen)));
  off += headLen;

  const blocks = [];
  const metadata = [];
  while (!headerOnly) {
    const type = tag4();
    if (type === "tail") break;
    off += 4;
    const len = view.getInt32(off); off += 4;
    if (type === "ppak") {
      const cols = view.getInt32(off);
      const rows = view.getInt32(off + 4);
      const grids = view.getInt32(off + 8);
      const scale = Math.pow(10, view.getFloat32(off + 12));
      const packed = new Uint8Array(buffer, off + 16, len - 16);
      const values = new Float32Array(cols * rows * grids);
      varpackDecode(values, packed);
      undeltaPlane(values, cols, rows, grids);
      for (let i = 0; i < values.length; i++) values[i] /= scale;
      blocks.push(values);
      metadata.push({ type, cols, rows, grids, scaleFactor: scale });
    } else {
      throw new Error(`unhandled block type '${type}'`); // qpak etc. — add if ever seen
    }
    off += len;
  }
  return { header, blocks, metadata };
}

// Prefix-coded signed varints; 0xFF introduces a run of NaNs (count = next byte + 1).
export function varpackDecode(out, bytes) {
  let i = 0, j = 0;
  while (i < bytes.length) {
    const b0 = bytes[i++];
    let v;
    if (b0 < 0x80) {                        // 0xxxxxxx : 7-bit
      v = (b0 << 25) >> 25;
    } else if (b0 < 0xc0) {                 // 10xxxxxx : 14-bit
      v = ((b0 << 26) >> 18) | bytes[i++];
    } else if (b0 < 0xe0) {                 // 110xxxxx : 21-bit
      v = ((b0 << 27) >> 11) | (bytes[i++] << 8) | bytes[i++];
    } else if (b0 < 0xf0) {                 // 1110xxxx : 28-bit
      v = ((b0 << 28) >> 4) | (bytes[i++] << 16) | (bytes[i++] << 8) | bytes[i++];
    } else if (b0 === 0xff) {               // NaN run
      for (let run = bytes[i++] + 1; run > 0; run--) out[j++] = NaN;
      continue;
    } else if (b0 < 0xf8) {                 // 11110xxx : raw int32 follows
      v = (bytes[i++] << 24) | (bytes[i++] << 16) | (bytes[i++] << 8) | bytes[i++];
    } else {
      throw new Error(`varpack: unsupported prefix 0x${b0.toString(16)}`);
    }
    out[j++] = v;
  }
  return out;
}

// Values are residuals against predictor left + above - aboveLeft (per z-grid),
// with NaN-tolerant fallbacks matching the reference behaviour.
export function undeltaPlane(v, cols, rows, grids) {
  for (let z = 0; z < grids; z++) {
    const k = z * cols * rows;
    for (let x = 1; x < cols; x++) {
      const p = v[k + x - 1];
      v[k + x] += p === p ? p : 0;
    }
    for (let y = 1; y < rows; y++) {
      const j = k + y * cols;
      const q = v[j - cols];
      v[j] += q === q ? q : 0;
      for (let x = 1; x < cols; x++) {
        const i = j + x;
        const a = v[i - 1], b = v[i - cols], c = v[i - cols - 1];
        const p = a + b - c;
        v[i] += p === p ? p : a === a ? a : b === b ? b : c === c ? c : 0;
      }
    }
  }
  return v;
}