listCraftFolders.js

#!/usr/bin/env node

// Placed in the public domain

// list folders in primary Craft space
// I've combined this into one file for the gist
// The top reads the Craft DB into memory (takes about 380ms for 50k blocks)
// The end looks up and prints the folder names

// You can reuse the top bits for other purposes.

// This script will need to be adjusted when craft updates to Realm 6.

let {readdirSync, readFileSync} = require("fs");

let assert = (cond, msg) => { if (!cond) throw Error(msg || "Assert"); };

function read_database(rfile2, translateRefs = true) {
  function readBlob(ref) {
    assert(!(ref & 1), `bad ref ${ref}`);
    let size = (buf[ref + 5] << 16) + (buf[ref + 6] << 8) + buf[ref + 7];
    let h = buf[ref + 4];
    let width = 1 << (h & 7) >> 1;
    let wtype = (h & 24) >> 3;
    assert(wtype === 2);
    assert(width === 1);
    return buf.toString("utf8", ref + 8, ref + 8 + size);
  }
  function readStringArray(ref) {
    assert(!(ref & 1), `bad ref ${ref}`);
    let size = (buf[ref + 5] << 16) + (buf[ref + 6] << 8) + buf[ref + 7];
    let h = buf[ref + 4];
    let width = 1 << (h & 7) >> 1;
    let wtype = (h & 24) >> 3;
    assert(wtype === 1);
    let rval = [];
    for (let i = 0; i < size; i++) {
      let s = ref + 8 + width * i;
      let e = s + width - buf[s + width - 1] - 1;
      rval.push(buf.toString("utf8", s, e));
    }
    return rval;
  }
  const getFlags = (ref) => buf[ref + 4] >> 5;
  function readArray(ref) {
    assert(!(ref & 1), `bad ref ${ref}`);
    let size = (buf[ref + 5] << 16) + (buf[ref + 6] << 8) + buf[ref + 7];
    let h = buf[ref + 4];
    let width = 1 << (h & 7) >> 1;
    let wtype = (h & 24) >> 3;
    assert(wtype === 0);
    let rval = [];
    for (let i = 0; i < size; i++) {
      if (width === 0)
        rval.push(0);
      else if (width === 1)
        rval.push(1 & buf[ref + 8 + i / 8 | 0] >> i % 8);
      else if (width === 2)
        rval.push(3 & buf[ref + 8 + i / 4 | 0] >> 2 * (i % 3));
      else if (width === 4)
        rval.push(15 & buf[ref + 8 + i / 2 | 0] >> 4 * (i % 2));
      else if (width === 8)
        rval.push(buf[ref + 8 + i]);
      else if (width === 16)
        rval.push(buf.readUInt16LE(ref + 8 + i * 2));
      else if (width === 32)
        rval.push(buf.readUInt32LE(ref + 8 + i * 4));
      else
        assert(false, `width ${width} ints not handled`);
    }
    return rval;
  }
  const readers = {
    int(acc, ref) {
      assert(getFlags(ref) == 0);
      acc.push(...readArray(ref));
    },
    bool(acc, ref) {
      assert(getFlags(ref) == 0);
      for (let x of readArray(ref))
        acc.push(x == 1);
    },
    linklist(acc, ref) {
      assert(getFlags(ref) == 2);
      readArray(ref).forEach((v) => acc.push(v == 0 ? [] : readArray(v)));
    },
    timestamp(acc, ref) {
      assert(getFlags(ref) == 2);
      scanBPTree(acc, readArray(ref)[0], (acc2, ref2) => readArray(ref2).slice(1).forEach((x) => acc2.push(x)));
    },
    string(acc, ref) {
      let flags = getFlags(ref);
      if (flags == 2) {
        let arr = readArray(ref);
        let ends = readArray(arr[0]);
        let blob = readBlob(arr[1]);
        let s = 0;
        ends.forEach((e) => {
          acc.push(blob.slice(s, e - 1));
          s = e;
        });
      } else if (flags === 3) {
        for (let r of readArray(ref)) {
          acc.push(r == 0 ? null : readBlob(r).slice(0, -1));
        }
      } else if (flags === 0) {
        acc.push(...readStringArray(ref));
      }
    }
  };
  function scanBPTree(acc, ref, reader) {
    let flags = getFlags(ref);
    if (flags & 4) {
      readArray(ref).slice(1, -1).forEach((ref2) => scanBPTree(acc, ref2, reader));
    } else {
      reader(acc, ref);
    }
  }
  const colTypes = {0: "int", 1: "bool", 2: "string", 8: "timestamp", 13: "linklist"};
  function readTable(name, ref) {
    let arr = readArray(ref);
    let spec = readArray(arr[0]);
    let types = readArray(spec[0]);
    let names = readStringArray(spec[1]);
    let attrs = readArray(spec[2]);
    let subspecs = spec.length > 3 && readArray(spec[3]);
    let crefs = readArray(arr[1]);
    let cix = 0;
    let six = 0;
    let cols = [];
    let refs = {};
    for (let i = 0; i < names.length; i++) {
      let cname = names[i];
      let ct = colTypes[types[i]];
      if (subspecs && ct == "linklist") {
        refs[cname] = subspecs[six++] >> 1;
      }
      let col = [];
      scanBPTree(col, crefs[cix], readers[ct]);
      cols.push(col);
      cix++;
      if (attrs[i] & 1)
        cix++;
    }
    let rows = [];
    for (let i = 0; i < cols[0].length; i++) {
      let row = {};
      for (let j = 0; j < names.length; j++) {
        row[names[j]] = cols[j][i];
      }
      rows.push(row);
    }
    return {name, rows, refs};
  }
  function readTop(ref) {
    let [rNames, rTables] = readArray(ref);
    let names = readStringArray(rNames);
    let trefs = readArray(rTables);
    return names.map((name, i) => readTable(name, trefs[i]));
  }
  function get_topref(buf2) {
    let magic = buf2.readInt32LE(16);
    assert(magic == 0x42442d54, "bad magic");
    let sel = buf2.readUInt8(23) & 1;
    let topref2 = buf2.readInt32LE(sel * 8);
    return topref2;
  }
  let buf = readFileSync(rfile2);
  let topref = get_topref(buf);
  let tables = readTop(topref);
  let pks = {};
  let db2 = {};
  for (let table of tables) {
    if (table.name == "pk") {
      for (let {pk_table, pk_property} of table.rows)
        pks[pk_table] = pk_property;
    } else if (table.name.startsWith("class_")) {
      db2[table.name.slice(6)] = table.rows;
    }
  }
  if (translateRefs)
    for (let table of tables) {
      for (let k in table.refs) {
        let dest = tables[table.refs[k]];
        let dpk = pks[dest.name.slice(6)];
        console.log(table.name, k, "->", dest.name, dpk);
        for (let item of table.rows) {
          let value = item[k];
          if (value && value.length) {
            for (let i = 0; i < value.length; i++)
              value[i] = dest.rows[value[i]][dpk];
          }
        }
      }
    }
  return db2;
}

// Code to dump folder list

function dumpFolders(db2) {
  function print(id, prefix = "") {
    for (let f of tree[id] || []) {
      console.log(f.id, prefix + f.name);
      print(f.id, prefix + "  ");
    }
  }
  let tree = {};
  for (let f of db2.FolderDataModel) {
    let pid = f.parentFolderId;
    tree[pid] = tree[pid] || [];
    tree[pid].push(f);
  }
  print("");
}

// Code to select the space file we want. There seems to be a primary one
// whose filename is the prefix of the other, secondary spaces, so we go
// for the shortest filename.

let base = `${process.env.HOME}/Library/Containers/com.lukilabs.lukiapp/Data/Library/Application Support/com.lukilabs.lukiapp`;
let realmFileName;
for (let f of readdirSync(base)) {
  if (f.startsWith("LukiMain") && f.endsWith(".realm") && (!realmFileName || realmFileName.length > f.length))
    realmFileName = f;
}
let rfile = `${base}/${realmFileName}`;
let db = read_database(rfile, false);
dumpFolders(db);