timm · June 23, 2026 00:39
diff --git a/,skape.md b/,skape.md
diff --git a/.gitignore b/.gitignore
 __pycache__/
 *.pyc
 dist/
 build/
 *.egg-info/
diff --git a/banner.txt b/banner.txt
     _                    
 ___| | ____ _ _ __   ___ 
 / __| |/ / _` | '_ \ / _ \
 \__ \   < (_| | |_) |  __/
 |___/_|\_\__,_| .__/ \___|
              |_|         
 FastMap landscape active learner. http://tiny.cc/skape
 "label a few, project the rest, keep the best 2/3."
diff --git a/elites.png b/elites.png
diff --git a/elites_all.png b/elites_all.png
diff --git a/forest.py b/forest.py
 #!/usr/bin/env python3 -B
 """forest.py: a stable FFT forest over unsupervised background cuts.

 Built on skape's landscape learner. Per rep: split a pool train/test;
 landscape acquires <=50 labels on train; discretize the WHOLE train
 pool unsupervised (Sym->value, Num->16 logistic-CDF bins) = the
 background. Then grow 32 attribute-bagged depth-4 binary trees on the
 ~50 labels, using only background cuts, scoring splits by ENTROPY of
 the y-bins (so Num and Sym sit on one scale -- no m2-vs-entropy). Fan
 each tree to <=16 FFTs, keep the lowest-median-leaf-d2h one; min over
 the 32; sort test by it; top-`check` win. Mean over `repeats`.

 Rule learner sees every X value, but y only for the locally-labelled.

 eg: python3 -B forest.py -file ../optimiz/auto93.csv   (needs skape.py)
 """
 import sys, random
 from statistics import median
 import skape as S
 from skape import (o, isa, Sym, Num, add, csv, Data, disty, norm,
                   spread, some, clone, wins, pick, landscape)

 the = S.the
 the.trees, the.attrs, the.depth = 32, 0.75, 4   # forest knobs
 the.nbins, the.ybins, the.guard = 16, 16, 2

 # ---- unsupervised discretization (the "background") ----------
 def cutid(col, v):
  "Bin v: a Sym's value, or 1 of nbins logistic-CDF bins (Num)."
  if v == "?" or isa(col, Sym): return v
  return min(the.nbins-1, int(the.nbins * norm(col, v)))

 def ybin(y): return min(the.ybins-1, int(the.ybins * y))

 def ent(ys):
  "Entropy of the y-bins of a list of disty values."
  c = Sym()
  for y in ys: add(c, ybin(y))
  return spread(c)

 def labelled(data, rows):
  "Each labelled row -> (ids{at:cutid}, disty); ids use background cols."
  return [({at: cutid(data.cols[at], r[at]) for at in data.x}, disty(data, r))
          for r in rows]

 # ---- grow one attribute-bagged binary tree -------------------
 def bestcut(data, items, attrs, leaf):
  "Lowest entropy split (at, k, sym, yes, no) over the sampled attrs."
  best = None
  for at in attrs:
    sym = isa(data.cols[at], Sym)
    ks  = sorted({ids[at] for ids, _ in items if ids[at] != "?"})
    for k in (ks if sym else ks[:-1]):           # Num: drop the top bin
      yes = [it for it in items
             if it[0][at] != "?" and (it[0][at] == k if sym else it[0][at] <= k)]
      no  = [it for it in items if it not in yes]
      if len(yes) >= leaf and len(no) >= leaf:
        s = (len(yes)*ent([y for _, y in yes])
             + len(no)*ent([y for _, y in no])) / len(items)
        if best is None or s < best[0]: best = (s, at, k, sym, yes, no)
  return best

 def grow(data, items, attrs, depth):
  "Binary tree; every node carries its median d2h (mu) and n."
  ys = [y for _, y in items]
  t = o(at=None, mu=median(ys), n=len(items))
  if depth > 0 and len(items) >= 2*the.leaf:
    if cut := bestcut(data, items, attrs, the.leaf):
      _, at, k, sym, yes, no = cut
      t.at, t.k, t.sym = at, k, sym
      t.left  = grow(data, yes, attrs, depth-1)
      t.right = grow(data, no,  attrs, depth-1)
  return t

 # ---- fan a tree into FFTs; keep the best ---------------------
 def ffts(t):
  "Each level, one child exits as a leaf -> the 2^depth FFTs."
  if t.at is None:
    yield t
  else:
    for yes in (False, True):                    # which child exits
      ex, cont = (t.left, t.right) if yes else (t.right, t.left)
      lf = o(at=None, mu=ex.mu, n=ex.n)
      for rest in ffts(cont):
        yield o(at=t.at, k=t.k, sym=t.sym, yes=yes, left=lf, right=rest)

 def leafmin(f):
  "The lowest median-d2h leaf of an FFT (leaves with n>=guard)."
  ms, t = [], f
  while t.at is not None: ms.append(t.left); t = t.right
  ms.append(t)
  ms = [lf.mu for lf in ms if lf.n >= the.guard]
  return min(ms) if ms else 1e32

 def forest(data, items):
  "32 attribute-bagged trees -> the single best FFT (lowest leaf d2h)."
  m, best = max(1, int(the.attrs*len(data.x))), None
  for _ in range(the.trees):
    f = min(ffts(grow(data, items, some(data.x, m), the.depth)), key=leafmin)
    if best is None or leafmin(f) < leafmin(best): best = f
  return best

 # ---- predict (X-cuts only) + holdout win ---------------------
 def predict(data, f, row):
  "Walk an FFT to a leaf via background cuts; return its median d2h."
  while f.at is not None:
    v  = cutid(data.cols[f.at], row[f.at])
    go = True if v == "?" else (v == f.k if f.sym else v <= f.k)
    f  = f.left if go == f.yes else f.right
  return f.mu

 def holdout(data):
  win, out = wins(data), Num()
  for _ in range(the.repeats):
    rows = some(data.rows, the.cap)
    h = len(rows)//2
    d, test = clone(data, rows[:h]), rows[h:]
    items = labelled(d, landscape(d))            # <=50 labels, bg = train pool
    f = forest(d, items)
    best = pick(test, lambda r: predict(d, f, r), data)
    out = add(out, win(best))
  return out[1]

 def main():
  data = Data(csv(the.file))
  print(f"{holdout(data):.0f}\t{the.file.split('/')[-1]}")

 if __name__ == "__main__":
  for f, v in zip(sys.argv, sys.argv[1:]):
    if f[:1] == "-" and (k := f[1:]) in vars(the):
      old = getattr(the, k)
      setattr(the, k, v if isinstance(old, str) else type(old)(v))
  random.seed(the.seed)
  main()
diff --git a/Makefile b/Makefile
 # vim: ts=2 sw=2 sts=2 et :
 # knobs only; shared targets live in $(KONFIG)/Makefile
 KONFIG ?= ../konfig
 APP    := skape
 MAIN   := skape.py
 EXT    := py
 LANG   := python
 SRC    := *.py
 LINT   := ruff check skape.py
 TOOLS  := python3:run ruff:lint
 PKG    := python3 gawk ruff neovim tmux

 $(KONFIG)/Makefile:
 	@test -f $@ || { echo "missing konfig: git clone http://tiny.cc/konfig $(KONFIG)"; exit 1; }
 include $(KONFIG)/Makefile

 test: ## smoke-test (needs ../optimiz)
 	@python3 -B test_skape.py && echo "ok skape"

 push2pypi: ## build + upload to PyPI (needs ~/.pypirc account token)
 	@python3 -m build && python3 -m twine upload dist/*
 	@rm -rf dist build *.egg-info

 P ?= 8                      # xargs workers

 ~/tmp/konfig/skateper :
 	@mkdir -p $(@D)
 	@ls ../optimiz/*.csv | sort -R | xargs -P$(P) -I{} \
 	  python3 -B skape.py --file {} --data | tee $@
 	@gawk -v c=3 -f per.awk $@

diff --git a/pepper.png b/pepper.png
diff --git a/pepper2.png b/pepper2.png
diff --git a/per.awk b/per.awk
 # per.awk: print the 10,30,50,70,90th percentiles of column c
 # (default 1). gawk only (uses asort).
 #   ... | gawk -v c=3 -f per.awk      # c=3 -> skape median column
 BEGIN { if (!c) c = 1 }
 { a[NR] = $c }
 END { n = asort(a)
      for (p = 10; p <= 90; p += 20) {
        i = int(p/100 * n)
        printf "p%-2d %s\t", p, a[i ? i : 1] }
      print "" }
diff --git a/pyproject.toml b/pyproject.toml
 [project]
 name = "skape"
 version = "0.1.0"
 readme = ",skape.md"
 description = "FastMap landscape active learner: cheap labels -> a tiny tree"
 authors = [{name = "Tim Menzies", email = "timm@ieee.org"}]
 license = {text = "MIT"}
 requires-python = ">=3.12"
 dependencies = []            # stdlib only, by design

 [tool.setuptools]
 py-modules = ["skape"]

 [project.urls]
 Homepage = "http://tiny.cc/skape"

 # same terse house style as nuff -- 2-space indent, short names,
 # one-line colon bodies, grouped imports.
 [tool.ruff.lint]
 ignore = ["E701", "E702", "E731", "E741", "E401"]
diff --git a/raw.png b/raw.png
diff --git a/skape.py b/skape.py
 #!/usr/bin/env python3 -B
 import sys
 from math import exp
 from bisect import bisect_right
 from random import random as rand, sample, seed
 from types import SimpleNamespace as o
 isa = isinstance
 BIG, TINY = 1e32, 1e-32

 the = o(seed=1234567891, grow=4, keep=0.66, budget=50, cap=1024, 
        check=5, leaf=3, repeats=20, eps=0.1, maxd=4, fast=100, 

        slow=10, few=6, pos=0, file="../optimiz/auto93.csv")

 #-- cols -------------------------------------------------------
 Sym = dict
 def Num(n=0, mu=0, m2=0): return (n, mu, m2)
 def mu_(x): return x[1]

 def sd(num):
  n, mu, m2 = num
  return 0 if n < 2 else (max(0, m2) / (n-1)) ** 0.5

 def welford(num, v, inc=1):
  n, mu, m2 = num
  if (n := n + inc) <= 0: return Num()
  d = v - mu; mu += inc * d / n
  return n, mu, m2 + inc * d * (v - mu)

 def norm(num, v):
  if v == "?": return v
  z = (v - mu_(num)) / (sd(num) + TINY)
  return 1 / (1 + exp(-1.7 * max(-3, min(3, z))))

 #-- Data -------------------------------------------------------
 def Data(src=None):
  src = iter(src or [])
  data = o(names=next(src), cols={}, x=[], y=[], goal={},
           klass=None, protect=[], rows=[])
  return adds(src, roles(data))

 def roles(data):
  for at, s in enumerate(data.names):
    data.cols[at] = Num() if s[0].isupper() else Sym()
    if s[-1] == "X": continue
    if s[-1] == "~": data.protect.append(at); continue   # held aside
    if s[-1] in "+-!":
      data.y.append(at); data.goal[at] = s[-1] == "+"
      if s[-1] == "!": data.klass = at
    else: data.x.append(at)
  return data

 def add(i, v, inc=1):
  if isa(i, o):
    (i.rows.append if inc == 1 else i.rows.remove)(v)
    for at in i.cols: i.cols[at] = add(i.cols[at], v[at], inc)
    return i
  if v == "?": return i
  if isa(i, Sym): i[v] = i.get(v, 0) + inc; return i
  return welford(i, v, inc)

 def adds(src, i=None):
  if i is None: i = Num()
  for v in src: i = add(i, v)
  return i

 def clone(data, src=[]): return Data([data.names] + src)

 #-- dist -------------------------------------------------------
 def minkowski(vals, p=2):
  total, n = 0, 0
  for v in vals: total += v ** p; n += 1
  return (total / (n or 1)) ** (1 / p)

 def disty(data, row, **kw):
  return minkowski(
           (abs(norm(data.cols[at], row[at]) - data.goal[at])
           for at in data.y if row[at] != "?"), **kw)

 def distx(data, r1, r2, **kw):
  return minkowski((gap(data.cols[at], r1[at], r2[at])
                    for at in data.x), **kw)

 def gap(col, u, v):
  if u == v == "?": return 1
  if isa(col, Sym): return u != v
  u, v = norm(col, u), norm(col, v)
  if u == "?": u = 1 if v < 0.5 else 0
  if v == "?": v = 1 if u < 0.5 else 0
  return abs(u - v)


 #-- sample ------------------------------------------------------
 def project(rows, d, y):
  far = lambda r: max(rows, key=lambda z: d(z, r))
  east = far(rows[0]); west = far(east)
  if y(east) < y(west): east, west = west, east
  c = d(east, west) + TINY
  return lambda r: (d(east, r)**2 + c*c - d(west, r)**2) / (2*c)

 def landscape(data):
  d = lambda r1, r2: distx(data, r1, r2)
  y, ys = memo(lambda r: disty(data, r))
  pool = shuffle(data.rows)
  while len(ys) < the.budget-the.grow and len(pool)>=2*the.leaf:
    lab, k = [], 0
    for r in pool:
      if r in ys: lab.append(r)
      elif k < the.grow: y(r); lab.append(r); k += 1
    n = max(1, int((1-the.keep)*len(pool)))
    pool = sorted(pool, key=project(lab, d, y))[n:]
  return sorted(ys, key=y)

 #-- bands -------------------------------------------------------
 def has(v, lo, hi): return v == "?" or lo <= v <= hi
 def F(xs, v): return bisect_right(xs, v) / len(xs)

 def splits(b, r, lo=-BIG, hi=BIG, d=0):
  cut = [v for v in b + r if lo < v < hi]
  v = max(cut, key=lambda v: abs(F(b, v)-F(r, v)), default=None)
  if v is None or abs(F(b,v)-F(r,v)) < the.eps or d >= the.maxd:
    yield lo,hi,(F(b,hi) - F(b,lo)) - (F(r,hi) - F(r,lo)); return
  yield from splits(b, r, lo, v, d+1)
  yield from splits(b, r, v, hi, d+1)

 def bands(data, best, rest):
  nb, nr = len(best), len(rest)
  for at in data.x:
    if isa(data.cols[at], Sym):
      fb, fr = Sym(), Sym()
      for row in best: add(fb, row[at])
      for row in rest: add(fr, row[at])
      for v in fb | fr: 
        yield fb.get(v,0)/nb - fr.get(v,0)/nr, at, v, v
    else:
      b = sorted(row[at] for row in best if row[at] != "?")
      r = sorted(row[at] for row in rest if row[at] != "?")
      if b and r:
        for lo, hi, w in splits(b, r): yield w, at, lo, hi

 #-- rules -------------------------------------------------------
 def esample(bnd, k):
  key = lambda x:rand()**(1 / (abs(x[0]) or TINY))
  return sorted(bnd, key=key)[-k:]

 def rule(sub):
  g={}
  for w,at,lo,hi in sub: g[at] = g.get(at,[]) + [(lo,hi)]
  return g

 def selects(g, row):
  return all(any(has(row[at], lo, hi) for lo, hi in v) 
             for at, v in g.items())

 def learn(data, lab, split, cost):      # w: sum-weights filter, rescore on lab
  bnd= [x for x in bands(data, *split(data, lab)) if abs(x[0]) >= 0.05]
  lst= [esample(bnd, 1+int(rand()*5)) for _ in range(the.fast)]
  top= sorted(lst,key=lambda s: -sum(w for w,*_ in s))[:the.slow]
  return rule(min(top, key=lambda s: cost(data, lab, rule(s))))

 def holdout(data, prep, split, cost, score):
  out = []
  for _ in range(the.repeats):
    rows = shuffle(data.rows)
    h = len(rows)//2
    lab = prep(data, rows[:h])
    g = learn(data, lab, split, cost)
    out.append(score(data, g, rows[h:]))
  return out

 def wins(data):
  ys = sorted(disty(data, r) for r in data.rows)
  ten = len(ys)//10
  lo, med, sd = ys[0], ys[5*ten], (ys[9*ten] - ys[ten])/2.56
  def f(row):
    v = disty(data, row)
    if v < lo + 0.35*sd: v = lo
    return max(-100, int(100*(1 - (v-lo)/(med-lo + TINY))))
  return f

 def edges(lab, key):
  lab = sorted(lab, key=key)
  return lab[:the.few], lab[-the.few:]

 def meanof(rows, y): 
  return mu_(adds(map(y, rows))) if rows else BIG

 def chosen(g, test, y):                 # the rule's pick: best of check selected
  return min(some([r for r in test if selects(g,r)] or test, the.check), key=y)

 def rskape(data):
  win = wins(data)
  y = lambda r: disty(data, r)
  return holdout(data,
    lambda d,tr: landscape(clone(d, some(tr, the.cap))),
    lambda d,lab: edges(lab, y),
    lambda d,lab,g: meanof([r for r in lab if selects(g,r)], y),
    lambda d,g,test: win(chosen(g, test, y)))

 def metrics(data, g, rows, pos):        # rule as classifier -> pd,pf,prec,spd
  got  = [selects(g, r) for r in rows]
  want = [r[data.klass] == pos for r in rows]
  tp = sum(w and h for w,h in zip(want,got))
  fp = sum((not w) and h for w,h in zip(want,got))
  fn = sum(w and not h for w,h in zip(want,got)); tn = len(rows)-tp-fp-fn
  pd, pf = tp/(tp+fn+TINY), fp/(fp+tn+TINY); prec = tp/(tp+fp+TINY)
  spd = 0                               # max selection-rate gap, sym groups
  for at in data.protect:
    if not isa(data.cols[at], Sym): continue
    grp = {}
    for r, h in zip(rows, got): grp[r[at]] = grp.get(r[at], []) + [h]
    rate = [sum(v)/len(v) for v in grp.values()]
    if rate: spd = max(spd, max(rate)-min(rate))
  return pd, pf, prec, spd

 def d2h(m): pd, pf, prec, spd = m; return minkowski([1-pd, pf, 1-prec, spd])

 def cskape(data):                       # -> (d2h, pd, pf, prec, spd) per rep
  k, pos = data.klass, the.pos
  return holdout(data,
    lambda d,tr: some(tr, the.cap),     # labels free: just sample
    lambda d,lab: ([r for r in lab if r[k]==pos],
                   [r for r in lab if r[k]!=pos]),
    lambda d,lab,g: d2h(metrics(d, g, lab, pos)),
    lambda d,g,test: (lambda m: (d2h(m), *m))(metrics(d, g, test, pos)))

 # -- MAP-Elites: illuminate the recall x pf grid (best d2h per cell) -
 def colvals(data, best, rest):          # at -> (sorted best vals, sorted rest)
  cv = {}
  for at in data.x:
    if not isa(data.cols[at], Sym):
      b = sorted(r[at] for r in best if r[at] != "?")
      r = sorted(r[at] for r in rest if r[at] != "?")
      if len(b) > 2 and r: cv[at] = (b, r)
  return cv

 def randcut(cv):                        # random range on a numeric col, with w
  at = sample(list(cv), 1)[0]; b, r = cv[at]; n = len(b)
  i, j = sorted([int(rand()*(n+1)), int(rand()*(n+1))])
  lo = -BIG if i == 0 else b[i-1]; hi = BIG if j >= n else b[j]
  return (F(b,hi)-F(b,lo)) - (F(r,hi)-F(r,lo)), at, lo, hi

 def elites(data, pos, seeds=800, iters=12000, G=14):
  rows = some(data.rows, the.cap)
  best = [r for r in rows if r[data.klass]==pos]
  rest = [r for r in rows if r[data.klass]!=pos]
  cv, pool = colvals(data, best, rest), list(bands(data, best, rest))
  draw = lambda: randcut(cv) if (cv and rand()<.6) else sample(pool, 1)[0]
  def mutate(sub):
    s = sub[:]
    if len(s) > 1 and rand() < .35: s.pop(int(rand()*len(s)))
    elif rand() < .7: s.append(draw())
    else: s[int(rand()*len(s))] = draw()
    return s
  arch = {}
  def place(sub):
    m = metrics(data, rule(sub), rows, pos); dh = d2h(m)
    c = (min(G-1,int(m[0]*G)), min(G-1,int(m[1]*G)))
    if c not in arch or dh < arch[c][0]: arch[c] = (dh, sub, m)
  for _ in range(seeds): place([draw() for _ in range(1+int(rand()*3))])
  for _ in range(iters): place(mutate(arch[sample(list(arch),1)[0]][1]))
  return arch                           # cell -> (d2h, sub, (pd,pf,prec,spd))


 #-- plumbing ----------------------------------------------------
 def thing(s):
  if (s[1:] if s[:1] == "-" else s).isdigit(): return int(s)
  try: return float(s)
  except ValueError: 
    return s == "True" or (s != "False" and s)

 def csv(file, clean=lambda s: s.partition("#")[0].split(",")):
  with open(file, encoding="utf-8") as f:
    for line in f:
      row = [x.strip() for x in clean(line)]
      if any(row): yield tuple(thing(x) for x in row)

 def shuffle(lst)    : return sample(lst, len(lst))
 def some(lst, k=512): return sample(lst, min(k, len(lst)))

 def memo(fn):
  cache = {}
  def f(r):
    if r not in cache: cache[r] = fn(r)
    return cache[r]
  return f, cache

 def pcts(xs):
  xs = sorted(xs)
  return " ".join(f"{xs[int(p/100*len(xs))]:>4}"
                  for p in (10,30,50,70,90))

 #-- main -------------------------------------------------------
 def main():
  print(f"{pcts(rskape(Data(csv(the.file))))}",end=" ")
  print(f"{the.file.split('/')[-1][:-4]}")

 def test__the():  print(the)
 def test__csv():  [print(r) for r in csv(the.file)]
 def test__data(): main()
 def test__class():                      # cskape d2h percentiles (low=good)
  xs = sorted(r[0] for r in cskape(Data(csv(the.file))))
  s = " ".join(f"{xs[int(p/100*len(xs))]:5.2f}" for p in (10,30,50,70,90))
  print(f"{s}  {the.file.split('/')[-1][:-4]}")

 if __name__ == "__main__":
  for k, v in zip(sys.argv, sys.argv[1:]):
    if hasattr(the, k[2:]): setattr(the, k[2:], thing(v))
  for k in sys.argv:
    seed(the.seed)
    if fn := vars().get('test__' + k[2:]): fn()
diff --git a/test_skape.py b/test_skape.py
 #!/usr/bin/env python3 -B
 """test_skape.py: smoke tests (needs ../optimiz/auto93.csv).

  python3 -B test_skape.py
 """
 import random
 import skape as S

 F = "../optimiz/auto93.csv"

 def test_data():
  "csv loads typed, hashable rows; columns get roles."
  d = S.Data(S.csv(F))
  assert len(d.rows) == 398 and len(d.y) == 3
  assert isinstance(d.rows[0], tuple)          # hashable

 def test_dist():
  "disty in 0..1, self-distx 0, distx symmetric."
  d = S.Data(S.csv(F)); a, b = d.rows[0], d.rows[1]
  assert 0 <= S.disty(d, a) <= 1
  assert S.distx(d, a, a) == 0
  assert abs(S.distx(d, a, b) - S.distx(d, b, a)) < 1e-9

 def test_landscape():
  "acquires ~budget labels, all from the data."
  d = S.Data(S.csv(F))
  lab = S.landscape(d)
  assert S.the.budget - 2*S.the.grow <= len(lab) <= S.the.budget
  assert all(r in d.rows for r in lab)

 def test_tree():
  "tree over the labels predicts a disty mean in 0..1."
  d = S.Data(S.csv(F))
  t = S.tree(d, S.landscape(d))
  assert 0 <= S.treePredict(t, d.rows[0]) <= 1

 def test_holdout():
  "the hold-out win score is a sane 0..100."
  assert 0 <= S.holdout(S.Data(S.csv(F))) <= 100

 if __name__ == "__main__":
  fails = 0
  for name, fn in sorted(vars().items()):
    if name.startswith("test_"):
      random.seed(S.the.seed)
      try: fn(); print("ok  ", name)
      except Exception as e: fails += 1; print("FAIL", name, e)
  raise SystemExit(fails)
	#!/usr/bin/env python3 -B
	"""forest.py: a stable FFT forest over unsupervised background cuts.

	Built on skape's landscape learner. Per rep: split a pool train/test;
	landscape acquires <=50 labels on train; discretize the WHOLE train
	pool unsupervised (Sym->value, Num->16 logistic-CDF bins) = the
	background. Then grow 32 attribute-bagged depth-4 binary trees on the
	~50 labels, using only background cuts, scoring splits by ENTROPY of
	the y-bins (so Num and Sym sit on one scale -- no m2-vs-entropy). Fan
	each tree to <=16 FFTs, keep the lowest-median-leaf-d2h one; min over
	the 32; sort test by it; top-`check` win. Mean over `repeats`.

	Rule learner sees every X value, but y only for the locally-labelled.

	eg: python3 -B forest.py -file ../optimiz/auto93.csv (needs skape.py)
	"""
	import sys, random
	from statistics import median
	import skape as S
	from skape import (o, isa, Sym, Num, add, csv, Data, disty, norm,
	spread, some, clone, wins, pick, landscape)

	the = S.the
	the.trees, the.attrs, the.depth = 32, 0.75, 4 # forest knobs
	the.nbins, the.ybins, the.guard = 16, 16, 2

	# ---- unsupervised discretization (the "background") ----------
	def cutid(col, v):
	"Bin v: a Sym's value, or 1 of nbins logistic-CDF bins (Num)."
	if v == "?" or isa(col, Sym): return v
	return min(the.nbins-1, int(the.nbins * norm(col, v)))

	def ybin(y): return min(the.ybins-1, int(the.ybins * y))

	def ent(ys):
	"Entropy of the y-bins of a list of disty values."
	c = Sym()
	for y in ys: add(c, ybin(y))
	return spread(c)

	def labelled(data, rows):
	"Each labelled row -> (ids{at:cutid}, disty); ids use background cols."
	return [({at: cutid(data.cols[at], r[at]) for at in data.x}, disty(data, r))
	for r in rows]

	# ---- grow one attribute-bagged binary tree -------------------
	def bestcut(data, items, attrs, leaf):
	"Lowest entropy split (at, k, sym, yes, no) over the sampled attrs."
	best = None
	for at in attrs:
	sym = isa(data.cols[at], Sym)
	ks = sorted({ids[at] for ids, _ in items if ids[at] != "?"})
	for k in (ks if sym else ks[:-1]): # Num: drop the top bin
	yes = [it for it in items
	if it[0][at] != "?" and (it[0][at] == k if sym else it[0][at] <= k)]
	no = [it for it in items if it not in yes]
	if len(yes) >= leaf and len(no) >= leaf:
	s = (len(yes)*ent([y for _, y in yes])
	+ len(no)*ent([y for _, y in no])) / len(items)
	if best is None or s < best[0]: best = (s, at, k, sym, yes, no)
	return best

	def grow(data, items, attrs, depth):
	"Binary tree; every node carries its median d2h (mu) and n."
	ys = [y for _, y in items]
	t = o(at=None, mu=median(ys), n=len(items))
	if depth > 0 and len(items) >= 2*the.leaf:
	if cut := bestcut(data, items, attrs, the.leaf):
	_, at, k, sym, yes, no = cut
	t.at, t.k, t.sym = at, k, sym
	t.left = grow(data, yes, attrs, depth-1)
	t.right = grow(data, no, attrs, depth-1)
	return t

	# ---- fan a tree into FFTs; keep the best ---------------------
	def ffts(t):
	"Each level, one child exits as a leaf -> the 2^depth FFTs."
	if t.at is None:
	yield t
	else:
	for yes in (False, True): # which child exits
	ex, cont = (t.left, t.right) if yes else (t.right, t.left)
	lf = o(at=None, mu=ex.mu, n=ex.n)
	for rest in ffts(cont):
	yield o(at=t.at, k=t.k, sym=t.sym, yes=yes, left=lf, right=rest)

	def leafmin(f):
	"The lowest median-d2h leaf of an FFT (leaves with n>=guard)."
	ms, t = [], f
	while t.at is not None: ms.append(t.left); t = t.right
	ms.append(t)
	ms = [lf.mu for lf in ms if lf.n >= the.guard]
	return min(ms) if ms else 1e32

	def forest(data, items):
	"32 attribute-bagged trees -> the single best FFT (lowest leaf d2h)."
	m, best = max(1, int(the.attrs*len(data.x))), None
	for _ in range(the.trees):
	f = min(ffts(grow(data, items, some(data.x, m), the.depth)), key=leafmin)
	if best is None or leafmin(f) < leafmin(best): best = f
	return best

	# ---- predict (X-cuts only) + holdout win ---------------------
	def predict(data, f, row):
	"Walk an FFT to a leaf via background cuts; return its median d2h."
	while f.at is not None:
	v = cutid(data.cols[f.at], row[f.at])
	go = True if v == "?" else (v == f.k if f.sym else v <= f.k)
	f = f.left if go == f.yes else f.right
	return f.mu

	def holdout(data):
	win, out = wins(data), Num()
	for _ in range(the.repeats):
	rows = some(data.rows, the.cap)
	h = len(rows)//2
	d, test = clone(data, rows[:h]), rows[h:]
	items = labelled(d, landscape(d)) # <=50 labels, bg = train pool
	f = forest(d, items)
	best = pick(test, lambda r: predict(d, f, r), data)
	out = add(out, win(best))
	return out[1]

	def main():
	data = Data(csv(the.file))
	print(f"{holdout(data):.0f}\t{the.file.split('/')[-1]}")

	if __name__ == "__main__":
	for f, v in zip(sys.argv, sys.argv[1:]):
	if f[:1] == "-" and (k := f[1:]) in vars(the):
	old = getattr(the, k)
	setattr(the, k, v if isinstance(old, str) else type(old)(v))
	random.seed(the.seed)
	main()
	# vim: ts=2 sw=2 sts=2 et :
	# knobs only; shared targets live in $(KONFIG)/Makefile
	KONFIG ?= ../konfig
	APP := skape
	MAIN := skape.py
	EXT := py
	LANG := python
	SRC := *.py
	LINT := ruff check skape.py
	TOOLS := python3:run ruff:lint
	PKG := python3 gawk ruff neovim tmux

	$(KONFIG)/Makefile:
	@test -f $@ \|\| { echo "missing konfig: git clone http://tiny.cc/konfig $(KONFIG)"; exit 1; }
	include $(KONFIG)/Makefile

	test: ## smoke-test (needs ../optimiz)
	@python3 -B test_skape.py && echo "ok skape"

	push2pypi: ## build + upload to PyPI (needs ~/.pypirc account token)
	@python3 -m build && python3 -m twine upload dist/*
	@rm -rf dist build *.egg-info

	P ?= 8 # xargs workers

	~/tmp/konfig/skateper :
	@mkdir -p $(@D)
	@ls ../optimiz/*.csv \| sort -R \| xargs -P$(P) -I{} \
	python3 -B skape.py --file {} --data \| tee $@
	@gawk -v c=3 -f per.awk $@
	# per.awk: print the 10,30,50,70,90th percentiles of column c
	# (default 1). gawk only (uses asort).
	# ... \| gawk -v c=3 -f per.awk # c=3 -> skape median column
	BEGIN { if (!c) c = 1 }
	{ a[NR] = $c }
	END { n = asort(a)
	for (p = 10; p <= 90; p += 20) {
	i = int(p/100 * n)
	printf "p%-2d %s\t", p, a[i ? i : 1] }
	print "" }
	[project]
	name = "skape"
	version = "0.1.0"
	readme = ",skape.md"
	description = "FastMap landscape active learner: cheap labels -> a tiny tree"
	authors = [{name = "Tim Menzies", email = "timm@ieee.org"}]
	license = {text = "MIT"}
	requires-python = ">=3.12"
	dependencies = [] # stdlib only, by design

	[tool.setuptools]
	py-modules = ["skape"]

	[project.urls]
	Homepage = "http://tiny.cc/skape"

	# same terse house style as nuff -- 2-space indent, short names,
	# one-line colon bodies, grouped imports.
	[tool.ruff.lint]
	ignore = ["E701", "E702", "E731", "E741", "E401"]
	#!/usr/bin/env python3 -B
	import sys
	from math import exp
	from bisect import bisect_right
	from random import random as rand, sample, seed
	from types import SimpleNamespace as o
	isa = isinstance
	BIG, TINY = 1e32, 1e-32

	the = o(seed=1234567891, grow=4, keep=0.66, budget=50, cap=1024,
	check=5, leaf=3, repeats=20, eps=0.1, maxd=4, fast=100,

	slow=10, few=6, pos=0, file="../optimiz/auto93.csv")

	#-- cols -------------------------------------------------------
	Sym = dict
	def Num(n=0, mu=0, m2=0): return (n, mu, m2)
	def mu_(x): return x[1]

	def sd(num):
	n, mu, m2 = num
	return 0 if n < 2 else (max(0, m2) / (n-1)) ** 0.5

	def welford(num, v, inc=1):
	n, mu, m2 = num
	if (n := n + inc) <= 0: return Num()
	d = v - mu; mu += inc * d / n
	return n, mu, m2 + inc * d * (v - mu)

	def norm(num, v):
	if v == "?": return v
	z = (v - mu_(num)) / (sd(num) + TINY)
	return 1 / (1 + exp(-1.7 * max(-3, min(3, z))))

	#-- Data -------------------------------------------------------
	def Data(src=None):
	src = iter(src or [])
	data = o(names=next(src), cols={}, x=[], y=[], goal={},
	klass=None, protect=[], rows=[])
	return adds(src, roles(data))

	def roles(data):
	for at, s in enumerate(data.names):
	data.cols[at] = Num() if s[0].isupper() else Sym()
	if s[-1] == "X": continue
	if s[-1] == "~": data.protect.append(at); continue # held aside
	if s[-1] in "+-!":
	data.y.append(at); data.goal[at] = s[-1] == "+"
	if s[-1] == "!": data.klass = at
	else: data.x.append(at)
	return data

	def add(i, v, inc=1):
	if isa(i, o):
	(i.rows.append if inc == 1 else i.rows.remove)(v)
	for at in i.cols: i.cols[at] = add(i.cols[at], v[at], inc)
	return i
	if v == "?": return i
	if isa(i, Sym): i[v] = i.get(v, 0) + inc; return i
	return welford(i, v, inc)

	def adds(src, i=None):
	if i is None: i = Num()
	for v in src: i = add(i, v)
	return i

	def clone(data, src=[]): return Data([data.names] + src)

	#-- dist -------------------------------------------------------
	def minkowski(vals, p=2):
	total, n = 0, 0
	for v in vals: total += v ** p; n += 1
	return (total / (n or 1)) ** (1 / p)

	def disty(data, row, **kw):
	return minkowski(
	(abs(norm(data.cols[at], row[at]) - data.goal[at])
	for at in data.y if row[at] != "?"), **kw)

	def distx(data, r1, r2, **kw):
	return minkowski((gap(data.cols[at], r1[at], r2[at])
	for at in data.x), **kw)

	def gap(col, u, v):
	if u == v == "?": return 1
	if isa(col, Sym): return u != v
	u, v = norm(col, u), norm(col, v)
	if u == "?": u = 1 if v < 0.5 else 0
	if v == "?": v = 1 if u < 0.5 else 0
	return abs(u - v)


	#-- sample ------------------------------------------------------
	def project(rows, d, y):
	far = lambda r: max(rows, key=lambda z: d(z, r))
	east = far(rows[0]); west = far(east)
	if y(east) < y(west): east, west = west, east
	c = d(east, west) + TINY
	return lambda r: (d(east, r)*2 + cc - d(west, r)*2) / (2c)

	def landscape(data):
	d = lambda r1, r2: distx(data, r1, r2)
	y, ys = memo(lambda r: disty(data, r))
	pool = shuffle(data.rows)
	while len(ys) < the.budget-the.grow and len(pool)>=2*the.leaf:
	lab, k = [], 0
	for r in pool:
	if r in ys: lab.append(r)
	elif k < the.grow: y(r); lab.append(r); k += 1
	n = max(1, int((1-the.keep)*len(pool)))
	pool = sorted(pool, key=project(lab, d, y))[n:]
	return sorted(ys, key=y)

	#-- bands -------------------------------------------------------
	def has(v, lo, hi): return v == "?" or lo <= v <= hi
	def F(xs, v): return bisect_right(xs, v) / len(xs)

	def splits(b, r, lo=-BIG, hi=BIG, d=0):
	cut = [v for v in b + r if lo < v < hi]
	v = max(cut, key=lambda v: abs(F(b, v)-F(r, v)), default=None)
	if v is None or abs(F(b,v)-F(r,v)) < the.eps or d >= the.maxd:
	yield lo,hi,(F(b,hi) - F(b,lo)) - (F(r,hi) - F(r,lo)); return
	yield from splits(b, r, lo, v, d+1)
	yield from splits(b, r, v, hi, d+1)

	def bands(data, best, rest):
	nb, nr = len(best), len(rest)
	for at in data.x:
	if isa(data.cols[at], Sym):
	fb, fr = Sym(), Sym()
	for row in best: add(fb, row[at])
	for row in rest: add(fr, row[at])
	for v in fb \| fr:
	yield fb.get(v,0)/nb - fr.get(v,0)/nr, at, v, v
	else:
	b = sorted(row[at] for row in best if row[at] != "?")
	r = sorted(row[at] for row in rest if row[at] != "?")
	if b and r:
	for lo, hi, w in splits(b, r): yield w, at, lo, hi

	#-- rules -------------------------------------------------------
	def esample(bnd, k):
	key = lambda x:rand()**(1 / (abs(x[0]) or TINY))
	return sorted(bnd, key=key)[-k:]

	def rule(sub):
	g={}
	for w,at,lo,hi in sub: g[at] = g.get(at,[]) + [(lo,hi)]
	return g

	def selects(g, row):
	return all(any(has(row[at], lo, hi) for lo, hi in v)
	for at, v in g.items())

	def learn(data, lab, split, cost): # w: sum-weights filter, rescore on lab
	bnd= [x for x in bands(data, *split(data, lab)) if abs(x[0]) >= 0.05]
	lst= [esample(bnd, 1+int(rand()*5)) for _ in range(the.fast)]
	top= sorted(lst,key=lambda s: -sum(w for w,*_ in s))[:the.slow]
	return rule(min(top, key=lambda s: cost(data, lab, rule(s))))

	def holdout(data, prep, split, cost, score):
	out = []
	for _ in range(the.repeats):
	rows = shuffle(data.rows)
	h = len(rows)//2
	lab = prep(data, rows[:h])
	g = learn(data, lab, split, cost)
	out.append(score(data, g, rows[h:]))
	return out

	def wins(data):
	ys = sorted(disty(data, r) for r in data.rows)
	ten = len(ys)//10
	lo, med, sd = ys[0], ys[5ten], (ys[9ten] - ys[ten])/2.56
	def f(row):
	v = disty(data, row)
	if v < lo + 0.35*sd: v = lo
	return max(-100, int(100*(1 - (v-lo)/(med-lo + TINY))))
	return f

	def edges(lab, key):
	lab = sorted(lab, key=key)
	return lab[:the.few], lab[-the.few:]

	def meanof(rows, y):
	return mu_(adds(map(y, rows))) if rows else BIG

	def chosen(g, test, y): # the rule's pick: best of check selected
	return min(some([r for r in test if selects(g,r)] or test, the.check), key=y)

	def rskape(data):
	win = wins(data)
	y = lambda r: disty(data, r)
	return holdout(data,
	lambda d,tr: landscape(clone(d, some(tr, the.cap))),
	lambda d,lab: edges(lab, y),
	lambda d,lab,g: meanof([r for r in lab if selects(g,r)], y),
	lambda d,g,test: win(chosen(g, test, y)))

	def metrics(data, g, rows, pos): # rule as classifier -> pd,pf,prec,spd
	got = [selects(g, r) for r in rows]
	want = [r[data.klass] == pos for r in rows]
	tp = sum(w and h for w,h in zip(want,got))
	fp = sum((not w) and h for w,h in zip(want,got))
	fn = sum(w and not h for w,h in zip(want,got)); tn = len(rows)-tp-fp-fn
	pd, pf = tp/(tp+fn+TINY), fp/(fp+tn+TINY); prec = tp/(tp+fp+TINY)
	spd = 0 # max selection-rate gap, sym groups
	for at in data.protect:
	if not isa(data.cols[at], Sym): continue
	grp = {}
	for r, h in zip(rows, got): grp[r[at]] = grp.get(r[at], []) + [h]
	rate = [sum(v)/len(v) for v in grp.values()]
	if rate: spd = max(spd, max(rate)-min(rate))
	return pd, pf, prec, spd

	def d2h(m): pd, pf, prec, spd = m; return minkowski([1-pd, pf, 1-prec, spd])

	def cskape(data): # -> (d2h, pd, pf, prec, spd) per rep
	k, pos = data.klass, the.pos
	return holdout(data,
	lambda d,tr: some(tr, the.cap), # labels free: just sample
	lambda d,lab: ([r for r in lab if r[k]==pos],
	[r for r in lab if r[k]!=pos]),
	lambda d,lab,g: d2h(metrics(d, g, lab, pos)),
	lambda d,g,test: (lambda m: (d2h(m), *m))(metrics(d, g, test, pos)))

	# -- MAP-Elites: illuminate the recall x pf grid (best d2h per cell) -
	def colvals(data, best, rest): # at -> (sorted best vals, sorted rest)
	cv = {}
	for at in data.x:
	if not isa(data.cols[at], Sym):
	b = sorted(r[at] for r in best if r[at] != "?")
	r = sorted(r[at] for r in rest if r[at] != "?")
	if len(b) > 2 and r: cv[at] = (b, r)
	return cv

	def randcut(cv): # random range on a numeric col, with w
	at = sample(list(cv), 1)[0]; b, r = cv[at]; n = len(b)
	i, j = sorted([int(rand()(n+1)), int(rand()(n+1))])
	lo = -BIG if i == 0 else b[i-1]; hi = BIG if j >= n else b[j]
	return (F(b,hi)-F(b,lo)) - (F(r,hi)-F(r,lo)), at, lo, hi

	def elites(data, pos, seeds=800, iters=12000, G=14):
	rows = some(data.rows, the.cap)
	best = [r for r in rows if r[data.klass]==pos]
	rest = [r for r in rows if r[data.klass]!=pos]
	cv, pool = colvals(data, best, rest), list(bands(data, best, rest))
	draw = lambda: randcut(cv) if (cv and rand()<.6) else sample(pool, 1)[0]
	def mutate(sub):
	s = sub[:]
	if len(s) > 1 and rand() < .35: s.pop(int(rand()*len(s)))
	elif rand() < .7: s.append(draw())
	else: s[int(rand()*len(s))] = draw()
	return s
	arch = {}
	def place(sub):
	m = metrics(data, rule(sub), rows, pos); dh = d2h(m)
	c = (min(G-1,int(m[0]G)), min(G-1,int(m[1]G)))
	if c not in arch or dh < arch[c][0]: arch[c] = (dh, sub, m)
	for _ in range(seeds): place([draw() for _ in range(1+int(rand()*3))])
	for _ in range(iters): place(mutate(arch[sample(list(arch),1)[0]][1]))
	return arch # cell -> (d2h, sub, (pd,pf,prec,spd))


	#-- plumbing ----------------------------------------------------
	def thing(s):
	if (s[1:] if s[:1] == "-" else s).isdigit(): return int(s)
	try: return float(s)
	except ValueError:
	return s == "True" or (s != "False" and s)

	def csv(file, clean=lambda s: s.partition("#")[0].split(",")):
	with open(file, encoding="utf-8") as f:
	for line in f:
	row = [x.strip() for x in clean(line)]
	if any(row): yield tuple(thing(x) for x in row)

	def shuffle(lst) : return sample(lst, len(lst))
	def some(lst, k=512): return sample(lst, min(k, len(lst)))

	def memo(fn):
	cache = {}
	def f(r):
	if r not in cache: cache[r] = fn(r)
	return cache[r]
	return f, cache

	def pcts(xs):
	xs = sorted(xs)
	return " ".join(f"{xs[int(p/100*len(xs))]:>4}"
	for p in (10,30,50,70,90))

	#-- main -------------------------------------------------------
	def main():
	print(f"{pcts(rskape(Data(csv(the.file))))}",end=" ")
	print(f"{the.file.split('/')[-1][:-4]}")

	def test__the(): print(the)
	def test__csv(): [print(r) for r in csv(the.file)]
	def test__data(): main()
	def test__class(): # cskape d2h percentiles (low=good)
	xs = sorted(r[0] for r in cskape(Data(csv(the.file))))
	s = " ".join(f"{xs[int(p/100*len(xs))]:5.2f}" for p in (10,30,50,70,90))
	print(f"{s} {the.file.split('/')[-1][:-4]}")

	if __name__ == "__main__":
	for k, v in zip(sys.argv, sys.argv[1:]):
	if hasattr(the, k[2:]): setattr(the, k[2:], thing(v))
	for k in sys.argv:
	seed(the.seed)
	if fn := vars().get('test__' + k[2:]): fn()
	#!/usr/bin/env python3 -B
	"""test_skape.py: smoke tests (needs ../optimiz/auto93.csv).

	python3 -B test_skape.py
	"""
	import random
	import skape as S

	F = "../optimiz/auto93.csv"

	def test_data():
	"csv loads typed, hashable rows; columns get roles."
	d = S.Data(S.csv(F))
	assert len(d.rows) == 398 and len(d.y) == 3
	assert isinstance(d.rows[0], tuple) # hashable

	def test_dist():
	"disty in 0..1, self-distx 0, distx symmetric."
	d = S.Data(S.csv(F)); a, b = d.rows[0], d.rows[1]
	assert 0 <= S.disty(d, a) <= 1
	assert S.distx(d, a, a) == 0
	assert abs(S.distx(d, a, b) - S.distx(d, b, a)) < 1e-9

	def test_landscape():
	"acquires ~budget labels, all from the data."
	d = S.Data(S.csv(F))
	lab = S.landscape(d)
	assert S.the.budget - 2*S.the.grow <= len(lab) <= S.the.budget
	assert all(r in d.rows for r in lab)

	def test_tree():
	"tree over the labels predicts a disty mean in 0..1."
	d = S.Data(S.csv(F))
	t = S.tree(d, S.landscape(d))
	assert 0 <= S.treePredict(t, d.rows[0]) <= 1

	def test_holdout():
	"the hold-out win score is a sane 0..100."
	assert 0 <= S.holdout(S.Data(S.csv(F))) <= 100

	if __name__ == "__main__":
	fails = 0
	for name, fn in sorted(vars().items()):
	if name.startswith("test_"):
	random.seed(S.the.seed)
	try: fn(); print("ok ", name)
	except Exception as e: fails += 1; print("FAIL", name, e)
	raise SystemExit(fails)