A naïve cache for holding color distances. Needs thread safety!

ColorDistanceCache.cs

/// <summary>
/// A cache for storing color distance matching results.
/// </summary>
/// <remarks>
/// The cache is limited to 646866 entries at 0.62MB.
/// </remarks>
private struct ColorDistanceCache
{
    private const int IndexBits = 5;
    private const int IndexAlphaBits = 3;
    private const int IndexCount = (1 << IndexBits) + 1;
    private const int IndexAlphaCount = (1 << IndexAlphaBits) + 1;
    private const int RgbShift = 8 - IndexBits;
    private const int AlphaShift = 8 - IndexAlphaBits;
    private const int TableLength = IndexCount * IndexCount * IndexCount * IndexAlphaCount;
    private short[] table;

    public static ColorDistanceCache Create()
    {
        ColorDistanceCache result = default;
        short[] entries = new short[TableLength];
        entries.AsSpan().Fill(-1);
        result.table = entries;

        return result;
    }

    [MethodImpl(InliningOptions.ShortMethod)]
    public void Add(Rgba32 rgba, byte index)
    {
        int r = rgba.R >> RgbShift;
        int g = rgba.G >> RgbShift;
        int b = rgba.B >> RgbShift;
        int a = rgba.A >> AlphaShift;
        int idx = GetPaletteIndex(r, g, b, a);
        this.table[idx] = index;
    }

    [MethodImpl(InliningOptions.ShortMethod)]
    public bool TryGetValue(Rgba32 rgba, out short match)
    {
        int r = rgba.R >> RgbShift;
        int g = rgba.G >> RgbShift;
        int b = rgba.B >> RgbShift;
        int a = rgba.A >> AlphaShift;
        int idx = GetPaletteIndex(r, g, b, a);
        match = this.table[idx];
        return match > -1;
    }

    [MethodImpl(InliningOptions.ShortMethod)]
    private static int GetPaletteIndex(int r, int g, int b, int a)
        => (r << ((IndexBits * 2) + IndexAlphaBits))
                + (r << (IndexBits + IndexAlphaBits + 1))
                + (g << (IndexBits + IndexAlphaBits))
                + (r << (IndexBits * 2))
                + (r << (IndexBits + 1))
                + (g << IndexBits)
                + ((r + g + b) << IndexAlphaBits)
                + r + g + b + a;
}