quantize_rgb_bc7_mode6.cpp

// Returns p bit that results in the lowest RGB quantization error.
inline int quantize_rgb_bc7_mode6(float r, float g, float b, float * out_qr, float * out_qg, float * out_qb) {
    float qr = 2 * trunc(r * 127.5f);
    float qg = 2 * trunc(g * 127.5f);
    float qb = 2 * trunc(b * 127.5f);

    float dr = 255 * r - qr;
    float dg = 255 * g - qg;
    float db = 255 * b - qb;

    // int e0 = dr * dr + dg * dg + db * db; // p=0 error
    // int e1 = (dr-1) * (dr-1) + (dg-1) * (dg-1) + (db-1) * (db-1); // p=1 error
    // int e1 = e0 - 2 * (dr + dg + db) + 3;
    // int p = e1 > e0;
    // int p = 3 - 2 * (dr + dg + db) > 0;
    int p = dr + dg + db >= 1.5f;

    qr += p;
    qg += p;
    qb += p;

    *out_qr = qr;
    *out_qg = qg;
    *out_qb = qb;
    return p;
}