jleedev · April 22, 2026 10:56
diff --git a/byte-viz.py b/byte-viz.py
 from collections.abc import Buffer
 import io
 import os
 from pathlib import Path
 import subprocess
 import textwrap

 import numpy as np
 import pandas as pd
 import polars as pl
 import pyarrow as pa
 import pyarrow.dataset as ds
 import matplotlib
 import matplotlib.pyplot as plt
 def iter_readinto(fh: io.BufferedReader, buf: Buffer):
    while (n := fh.readinto1(buf)):
        yield memoryview(buf)[:n] if n < len(buf) else buf
 def iter_frames(p):
    buf = bytearray(4*2**20)
    with open(p, 'rb') as fh:
        for chunk in iter_readinto(fh, buf):
            pb = pa.py_buffer(buf)
            arr = pa.Array.from_buffers(pa.uint16(), len(pb) / 2, [None, pb])
            rb = pa.record_batch({ 'data': arr })
            df = pl.from_arrow(rb)
            yield df
 def get_total_bytepairs(p):
    total = pl.DataFrame(schema=pl.Schema({ 'data': pl.UInt16, 'count': pl.UInt64 }))
    for s in iter_frames(p):
        count = s.to_series().value_counts()
        total = pl.concat([total, count]).group_by('data').agg(pl.sum('count'))
    return total
 def shape_grid(totals: pl.DataFrame):
    return (pl.select(data=pl.arange(65536))
        .join(totals, 'data', how='left')
        .select('count')
        .to_numpy()
        .reshape((256, -1)))
 def standard_scaler(data):
    return (data - np.nanmean(data)) / np.nanstd(data)
 def robust_scaler(data):
    qs = np.nanpercentile(data, [25, 50, 75])
    iqr = qs[2] - qs[0]
    if iqr == 0:
        iqr = 1
    return (data - qs[1]) / (1.5 * iqr)
 def byteplot(
    p: os.PathLike | str,
    transform=lambda d: d,
    scaler=robust_scaler,
    **kwargs,
 ) -> matplotlib.image.AxesImage:
    if not isinstance(p, os.PathLike):
        p = Path(p).expanduser()
    tit = subprocess.check_output(['file','-b', p]).decode()
    total = get_total_bytepairs(p)
    grid = shape_grid(total)
    kw = dict(cmap='turbo', vmin=-1, vmax=1)
    kw.update(kwargs)
    ax = plt.imshow(scaler(transform(grid)), **kw)
    if tit != 'data\n':
        plt.suptitle(textwrap.fill(tit, 40))
    return ax
diff --git a/output_100_0.png b/output_100_0.png
diff --git a/output_101_0.png b/output_101_0.png
diff --git a/output_102_0.png b/output_102_0.png
diff --git a/output_103_0.png b/output_103_0.png
diff --git a/output_104_0.png b/output_104_0.png
diff --git a/output_105_0.png b/output_105_0.png
diff --git a/output_106_0.png b/output_106_0.png
diff --git a/output_107_0.png b/output_107_0.png
diff --git a/output_108_0.png b/output_108_0.png
diff --git a/output_109_0.png b/output_109_0.png
diff --git a/output_10_0.png b/output_10_0.png
diff --git a/output_110_0.png b/output_110_0.png
diff --git a/output_111_0.png b/output_111_0.png
diff --git a/output_112_0.png b/output_112_0.png
diff --git a/output_113_0.png b/output_113_0.png
diff --git a/output_114_0.png b/output_114_0.png
diff --git a/output_115_0.png b/output_115_0.png
diff --git a/output_116_0.png b/output_116_0.png
diff --git a/output_117_0.png b/output_117_0.png
diff --git a/output_118_0.png b/output_118_0.png
diff --git a/output_119_0.png b/output_119_0.png
diff --git a/output_11_0.png b/output_11_0.png
diff --git a/output_120_0.png b/output_120_0.png
diff --git a/output_121_0.png b/output_121_0.png
diff --git a/output_122_0.png b/output_122_0.png
diff --git a/output_123_0.png b/output_123_0.png
diff --git a/output_124_0.png b/output_124_0.png
diff --git a/output_12_0.png b/output_12_0.png
diff --git a/output_13_0.png b/output_13_0.png
diff --git a/output_14_0.png b/output_14_0.png
diff --git a/output_15_0.png b/output_15_0.png
diff --git a/output_16_0.png b/output_16_0.png
diff --git a/output_17_0.png b/output_17_0.png
diff --git a/output_18_0.png b/output_18_0.png
diff --git a/output_19_0.png b/output_19_0.png
diff --git a/output_20_0.png b/output_20_0.png
diff --git a/output_21_0.png b/output_21_0.png
diff --git a/output_22_0.png b/output_22_0.png
diff --git a/output_23_0.png b/output_23_0.png
diff --git a/output_24_0.png b/output_24_0.png
diff --git a/output_25_0.png b/output_25_0.png
diff --git a/output_26_0.png b/output_26_0.png
diff --git a/output_27_0.png b/output_27_0.png
diff --git a/output_28_0.png b/output_28_0.png
diff --git a/output_29_0.png b/output_29_0.png
diff --git a/output_30_0.png b/output_30_0.png
diff --git a/output_31_0.png b/output_31_0.png
diff --git a/output_32_0.png b/output_32_0.png
diff --git a/output_33_0.png b/output_33_0.png
diff --git a/output_34_0.png b/output_34_0.png
diff --git a/output_35_0.png b/output_35_0.png
diff --git a/output_36_0.png b/output_36_0.png
diff --git a/output_37_0.png b/output_37_0.png
diff --git a/output_38_0.png b/output_38_0.png
diff --git a/output_39_0.png b/output_39_0.png
diff --git a/output_40_0.png b/output_40_0.png
diff --git a/output_41_0.png b/output_41_0.png
diff --git a/output_42_0.png b/output_42_0.png
diff --git a/output_43_0.png b/output_43_0.png
diff --git a/output_44_0.png b/output_44_0.png
diff --git a/output_45_0.png b/output_45_0.png
diff --git a/output_46_0.png b/output_46_0.png
diff --git a/output_47_0.png b/output_47_0.png
diff --git a/output_48_0.png b/output_48_0.png
diff --git a/output_49_0.png b/output_49_0.png
diff --git a/output_50_0.png b/output_50_0.png
diff --git a/output_51_0.png b/output_51_0.png
diff --git a/output_52_0.png b/output_52_0.png
diff --git a/output_53_0.png b/output_53_0.png
diff --git a/output_54_0.png b/output_54_0.png
diff --git a/output_55_0.png b/output_55_0.png
diff --git a/output_56_0.png b/output_56_0.png
diff --git a/output_57_0.png b/output_57_0.png
diff --git a/output_58_0.png b/output_58_0.png
diff --git a/output_59_1.png b/output_59_1.png
diff --git a/output_60_0.png b/output_60_0.png
diff --git a/output_61_0.png b/output_61_0.png
diff --git a/output_62_0.png b/output_62_0.png
diff --git a/output_63_0.png b/output_63_0.png
diff --git a/output_64_0.png b/output_64_0.png
diff --git a/output_65_0.png b/output_65_0.png
diff --git a/output_66_0.png b/output_66_0.png
diff --git a/output_67_0.png b/output_67_0.png
diff --git a/output_68_0.png b/output_68_0.png
diff --git a/output_69_0.png b/output_69_0.png
diff --git a/output_70_0.png b/output_70_0.png
diff --git a/output_71_0.png b/output_71_0.png
diff --git a/output_72_0.png b/output_72_0.png
diff --git a/output_73_0.png b/output_73_0.png
diff --git a/output_74_0.png b/output_74_0.png
diff --git a/output_75_0.png b/output_75_0.png
diff --git a/output_76_0.png b/output_76_0.png
diff --git a/output_77_0.png b/output_77_0.png
diff --git a/output_78_0.png b/output_78_0.png
diff --git a/output_79_0.png b/output_79_0.png
diff --git a/output_80_0.png b/output_80_0.png
diff --git a/output_81_0.png b/output_81_0.png
diff --git a/output_82_0.png b/output_82_0.png
diff --git a/output_83_0.png b/output_83_0.png
diff --git a/output_84_0.png b/output_84_0.png
diff --git a/output_85_0.png b/output_85_0.png
diff --git a/output_86_0.png b/output_86_0.png
diff --git a/output_87_0.png b/output_87_0.png
diff --git a/output_88_0.png b/output_88_0.png
diff --git a/output_89_0.png b/output_89_0.png
diff --git a/output_8_0.png b/output_8_0.png
diff --git a/output_90_1.png b/output_90_1.png
diff --git a/output_91_0.png b/output_91_0.png
diff --git a/output_92_0.png b/output_92_0.png
diff --git a/output_93_0.png b/output_93_0.png
diff --git a/output_94_0.png b/output_94_0.png
diff --git a/output_95_0.png b/output_95_0.png
diff --git a/output_96_0.png b/output_96_0.png
diff --git a/output_97_0.png b/output_97_0.png
diff --git a/output_98_0.png b/output_98_0.png
diff --git a/output_99_0.png b/output_99_0.png
diff --git a/output_9_0.png b/output_9_0.png
	from collections.abc import Buffer
	import io
	import os
	from pathlib import Path
	import subprocess
	import textwrap

	import numpy as np
	import pandas as pd
	import polars as pl
	import pyarrow as pa
	import pyarrow.dataset as ds
	import matplotlib
	import matplotlib.pyplot as plt
	def iter_readinto(fh: io.BufferedReader, buf: Buffer):
	while (n := fh.readinto1(buf)):
	yield memoryview(buf)[:n] if n < len(buf) else buf
	def iter_frames(p):
	buf = bytearray(42*20)
	with open(p, 'rb') as fh:
	for chunk in iter_readinto(fh, buf):
	pb = pa.py_buffer(buf)
	arr = pa.Array.from_buffers(pa.uint16(), len(pb) / 2, [None, pb])
	rb = pa.record_batch({ 'data': arr })
	df = pl.from_arrow(rb)
	yield df
	def get_total_bytepairs(p):
	total = pl.DataFrame(schema=pl.Schema({ 'data': pl.UInt16, 'count': pl.UInt64 }))
	for s in iter_frames(p):
	count = s.to_series().value_counts()
	total = pl.concat([total, count]).group_by('data').agg(pl.sum('count'))
	return total
	def shape_grid(totals: pl.DataFrame):
	return (pl.select(data=pl.arange(65536))
	.join(totals, 'data', how='left')
	.select('count')
	.to_numpy()
	.reshape((256, -1)))
	def standard_scaler(data):
	return (data - np.nanmean(data)) / np.nanstd(data)
	def robust_scaler(data):
	qs = np.nanpercentile(data, [25, 50, 75])
	iqr = qs[2] - qs[0]
	if iqr == 0:
	iqr = 1
	return (data - qs[1]) / (1.5 * iqr)
	def byteplot(
	p: os.PathLike \| str,
	transform=lambda d: d,
	scaler=robust_scaler,
	**kwargs,
	) -> matplotlib.image.AxesImage:
	if not isinstance(p, os.PathLike):
	p = Path(p).expanduser()
	tit = subprocess.check_output(['file','-b', p]).decode()
	total = get_total_bytepairs(p)
	grid = shape_grid(total)
	kw = dict(cmap='turbo', vmin=-1, vmax=1)
	kw.update(kwargs)
	ax = plt.imshow(scaler(transform(grid)), **kw)
	if tit != 'data\n':
	plt.suptitle(textwrap.fill(tit, 40))
	return ax
No results found