Skip to content

Instantly share code, notes, and snippets.

@Marc-B-Reynolds
Last active November 9, 2024 15:08
Show Gist options
  • Save Marc-B-Reynolds/e4fd22fd4a98616816e24ecb123fb2e9 to your computer and use it in GitHub Desktop.
Save Marc-B-Reynolds/e4fd22fd4a98616816e24ecb123fb2e9 to your computer and use it in GitHub Desktop.
UNORM8 to half float validation
/* -*- mode: c; -*- */
// multiply by recip: 10 aren't correctly rounded
print("multiply by 1/255");
for i from 0 to 255 do
{
cr = HP(i/255);
r = HP(i*0x1.01p-8);
if (cr != r) then {
print(" ",i);
};
};
// method of: https://marc-b-reynolds.github.io/math/2019/03/12/FpDiv.html
// requires FMA and denormals not flushing to zero
print("fma formulation");
k0 = 0x1.01p-8;
k1 = 0x1.00p-24;
for i from 0 to 255 do
{
cr = HP(i/255);
r = HP(k0*i + HP(k1*i));
if (cr != r) then {
print(" ",i);
};
};
// reworked to avoid denormals and FMAs are optional
// k0*(k1*i+i)
print("reworked: fma optional & no denormals");
k0 = 0x1.00p-8;
k1 = 0x1.01p-8;
for i from 0 to 255 do
{
cr = HP(i/255);
lo = HP(i*k1);
r = HP(k0 * HP(i+lo));
if (cr != r) then {
print(" ",i);
};
};
// k0*fma(k1,i,i)
print("reworked: using fma");
k0 = 0x1.00p-8;
k1 = 0x1.01p-8;
for i from 0 to 255 do
{
cr = HP(i/255);
r = HP(k0 * HP(k1*i+i));
if (cr != r) then {
print(" ",i);
};
};
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment