octahedral.glsl

// NOTE: inefficient, uses a lot of branches. only for demonstration.
// use under the creative commons zero license, any version
vec3 normal_encode(vec3 normal)
{
    // map onto octagon
    normal *= 1.0/(abs(normal.x) + abs(normal.y) + abs(normal.z));
    // project
    float n = normal.x;
    float m = normal.y;
    // shift projection for back layer
    if (normal.z < 0)
    {
        n += 0.5;
        m += 0.5;
        if (n > 1.0)
            n -= 2.0;
        if (m > 1.0)
            m -= 2.0;
    }
    
    uint ni = uint(round(n * 2047.0) + 2047.0);
    uint mi = uint(round(m * 2047.0) + 2047.0);
    
    return vec3(float(ni >> 4),
                float(((ni & uint(0xF)) << 4) | (mi >> 8)),
                float(mi & uint(0xFF))
               )/255.0;
}

vec3 normal_decode(vec3 normal)
{
    uint ix = uint(round(normal.x * 255.0));
    uint iy = uint(round(normal.y * 255.0));
    uint iz = uint(round(normal.z * 255.0));

    uint ni = (ix << 4) | ((iy >> 4) & uint(0xF));
    uint mi = iz | ((iy & uint(0xF)) << 8);

    float n = (float(ni) - 2047.0) / 2047.0;
    float m = (float(mi) - 2047.0) / 2047.0;

    // unproject front
    float x = n;
    float y = m;
    float z = 0.0;

    // shift x/y and flip z if back layer
    if (abs(n) + abs(m) > 1.0)
    {
        x -= 0.5;
        y -= 0.5;
        if (x < -1.0)
            x += 2.0;
        if (y < -1.0)
            y += 2.0;
        z = -(1.0 - (abs(x) + abs(y)));
    }
    else
    {
        z = 1.0 - (abs(x) + abs(y));
    }

    return normalize(vec3(x, y, z));
}