[libGDX] HQX (HQ2X, HQ3X, HQ4X) upscaling filter implementation (Java and Kotlin) using GLSL shaders.

hqnx.frag

// This float value should be defined from the compiling code.
// #define SCALE [2, 3, 4].0 

#ifdef GL_ES
#define PRECISION mediump
precision PRECISION float;
precision PRECISION int;
#else
#define PRECISION
#endif

uniform sampler2D u_texture;
uniform sampler2D u_lut;
uniform vec2 u_textureSize;

varying vec4 v_texCoord[4];

const mat3 YUV_MATRIX = mat3(0.299, 0.587, 0.114, -0.169, -0.331, 0.5, 0.5, -0.419, -0.081);
const vec3 YUV_THRESHOLD = vec3(48.0/255.0, 7.0/255.0, 6.0/255.0);
const vec3 YUV_OFFSET = vec3(0, 0.5, 0.5);

bool diff(vec3 yuv1, vec3 yuv2) {
    return any(greaterThan(abs((yuv1 + YUV_OFFSET) - (yuv2 + YUV_OFFSET)), YUV_THRESHOLD));
}

mat3 transpose(mat3 val) {
    mat3 result;
    result[0][1] = val[1][0];
    result[0][2] = val[2][0];
    result[1][0] = val[0][1];
    result[1][2] = val[2][1];
    result[2][0] = val[0][2];
    result[2][1] = val[1][2];
    return result;
}

void main() {
    vec2 fp = fract(v_texCoord[0].xy * u_textureSize);
    vec2 quad = sign(-0.5 + fp);
    mat3 yuv = transpose(YUV_MATRIX);

    float dx = v_texCoord[0].z;
    float dy = v_texCoord[0].w;
    vec3 p1  = texture2D(u_texture, v_texCoord[0].xy).rgb;
    vec3 p2  = texture2D(u_texture, v_texCoord[0].xy + vec2(dx, dy) * quad).rgb;
    vec3 p3  = texture2D(u_texture, v_texCoord[0].xy + vec2(dx, 0) * quad).rgb;
    vec3 p4  = texture2D(u_texture, v_texCoord[0].xy + vec2(0, dy) * quad).rgb;
    // Use mat4 instead of mat4x3 here to support GLES.
    mat4 pixels = mat4(vec4(p1, 0.0), vec4(p2, 0.0), vec4(p3, 0.0), vec4(p4, 0.0));

    vec3 w1  = yuv * texture2D(u_texture, v_texCoord[1].xw).rgb;
    vec3 w2  = yuv * texture2D(u_texture, v_texCoord[1].yw).rgb;
    vec3 w3  = yuv * texture2D(u_texture, v_texCoord[1].zw).rgb;

    vec3 w4  = yuv * texture2D(u_texture, v_texCoord[2].xw).rgb;
    vec3 w5  = yuv * p1;
    vec3 w6  = yuv * texture2D(u_texture, v_texCoord[2].zw).rgb;

    vec3 w7  = yuv * texture2D(u_texture, v_texCoord[3].xw).rgb;
    vec3 w8  = yuv * texture2D(u_texture, v_texCoord[3].yw).rgb;
    vec3 w9  = yuv * texture2D(u_texture, v_texCoord[3].zw).rgb;

    bvec3 pattern[3];
    pattern[0] =  bvec3(diff(w5, w1), diff(w5, w2), diff(w5, w3));
    pattern[1] =  bvec3(diff(w5, w4), false       , diff(w5, w6));
    pattern[2] =  bvec3(diff(w5, w7), diff(w5, w8), diff(w5, w9));
    bvec4 cross = bvec4(diff(w4, w2), diff(w2, w6), diff(w8, w4), diff(w6, w8));

    vec2 index;
    index.x = dot(vec3(pattern[0]), vec3(1, 2, 4)) +
    dot(vec3(pattern[1]), vec3(8, 0, 16)) +
    dot(vec3(pattern[2]), vec3(32, 64, 128));
    index.y = dot(vec4(cross), vec4(1, 2, 4, 8)) * (SCALE * SCALE) +
    dot(floor(fp * SCALE), vec2(1.0, SCALE));

    vec2 step = vec2(1.0) / vec2(256.0, 16.0 * (SCALE * SCALE));
    vec2 offset = step / vec2(2.0);
    vec4 weights = texture2D(u_lut, index * step + offset);
    float sum = dot(weights, vec4(1));
    vec3 res = (pixels * (weights / sum)).rgb;

    gl_FragColor.rgb = res;
}

hqnx.vert

#ifdef GL_ES
#define PRECISION mediump
precision PRECISION float;
precision PRECISION int;
#else
#define PRECISION
#endif

attribute vec4 a_position;
attribute vec2 a_texCoord0;

uniform vec2 u_textureSize;

varying vec4 v_texCoord[4];

void main() {
    gl_Position = a_position;

    vec2 ps = 1.0/u_textureSize;
    float dx = ps.x;
    float dy = ps.y;

    //   +----+----+----+
    //   |    |    |    |
    //   | w1 | w2 | w3 |
    //   +----+----+----+
    //   |    |    |    |
    //   | w4 | w5 | w6 |
    //   +----+----+----+
    //   |    |    |    |
    //   | w7 | w8 | w9 |
    //   +----+----+----+

    v_texCoord[0].zw = ps;
    v_texCoord[0].xy = a_texCoord0.xy;
    v_texCoord[1] = a_texCoord0.xxxy + vec4(-dx, 0, dx, -dy); //  w1 | w2 | w3
    v_texCoord[2] = a_texCoord0.xxxy + vec4(-dx, 0, dx,   0); //  w4 | w5 | w6
    v_texCoord[3] = a_texCoord0.xxxy + vec4(-dx, 0, dx,  dy); //  w7 | w8 | w9
}

HqnxEffect.java

import com.badlogic.gdx.Gdx;
import com.badlogic.gdx.files.FileHandle;
import com.badlogic.gdx.graphics.*;
import com.badlogic.gdx.graphics.VertexAttributes.Usage;
import com.badlogic.gdx.graphics.glutils.FrameBuffer;
import com.badlogic.gdx.graphics.glutils.ShaderProgram;
import com.badlogic.gdx.utils.Disposable;
import com.badlogic.gdx.utils.GdxRuntimeException;

/**
 * [HQnX](https://en.wikipedia.org/wiki/Pixel-art_scaling_algorithms#hqnx_family)
 * upscale algorithm GLSL implementation based on
 * [CrossVR](https://github.com/CrossVR/hqx-shader/tree/master/glsl) project.
 */
public class HqnxEffect implements Disposable {

    private static final int TEXTURE_HANDLE0 = 0;
    private static final int TEXTURE_HANDLE1 = 1;

    private static final String U_TEXTURE = "u_texture";
    private static final String U_LUT = "u_lut";
    private static final String U_TEXTURE_SIZE = "u_textureSize";

    private final ViewportQuadMesh mesh = new ViewportQuadMesh(
            new VertexAttribute(Usage.Position, 2, "a_position"),
            new VertexAttribute(Usage.TextureCoordinates, 2, "a_texCoord0"));

    private final ShaderProgram program;
    private final Texture lutTexture;
    private final int scaleFactor;

    private FrameBuffer dstBuffer;
    private int dstWidth = 0;
    private int dstHeight = 0;

    /** @param scaleFactor should be 2, 3 or 4 value. */
    public HqnxEffect(int scaleFactor) {
        if (scaleFactor < 2 || scaleFactor > 4) {
            throw new GdxRuntimeException("Scale factor should be 2, 3 or 4.");
        }

        program = compileShader(
                Gdx.files.internal("hqnx.vert"),
                Gdx.files.internal("hqnx.frag"),
                "#define SCALE " + scaleFactor + ".0");

        lutTexture = new Texture(Gdx.files.internal("hq" + scaleFactor + "x.png"));

        this.scaleFactor = scaleFactor;
    }

    @Override
    public void dispose() {
        mesh.dispose();
        program.dispose();
        lutTexture.dispose();
        if (dstBuffer != null) {
            dstBuffer.dispose();
        }
    }

    public void rebind() {
        program.bind();
        program.setUniformi(U_TEXTURE, TEXTURE_HANDLE0);
        program.setUniformi(U_LUT, TEXTURE_HANDLE1);
        program.setUniformf(U_TEXTURE_SIZE,
                dstWidth / (float)scaleFactor,
                dstHeight / (float)scaleFactor);
    }

    public void renderToScreen(Texture src) {
        validate(src);

        // Bind src buffer's texture as the primary one.
        lutTexture.bind(TEXTURE_HANDLE1);
        src.bind(TEXTURE_HANDLE0);

        program.bind();
        mesh.render(program);
    }

    public Texture renderToBuffer(Texture src) {
        validate(src);
        validateDstBuffer();

        // Bind src buffer's texture as the primary one.
        lutTexture.bind(TEXTURE_HANDLE1);
        src.bind(TEXTURE_HANDLE0);

        dstBuffer.begin();
        program.bind();
        mesh.render(program);
        dstBuffer.end();

        return dstBuffer.getColorBufferTexture();
    }

    private void validate(Texture src) {
        int targetWidth = src.getWidth() * scaleFactor;
        int targetHeight = src.getHeight() * scaleFactor;

        if (dstWidth != targetWidth || dstHeight != targetHeight) {
            dstWidth = targetWidth;
            dstHeight = targetHeight;
            rebind();
        }
    }

    private void validateDstBuffer() {
        if (dstBuffer == null || dstBuffer.getWidth() != dstWidth || dstBuffer.getHeight() != dstHeight) {
            if (dstBuffer != null) {
                dstBuffer.dispose();
            }
            dstBuffer = new FrameBuffer(Pixmap.Format.RGB888, dstWidth, dstHeight, false);
            dstBuffer.getColorBufferTexture().setFilter(
                    Texture.TextureFilter.Nearest,
                    Texture.TextureFilter.Nearest);
        }
    }

    /**
     * Encapsulates a fullscreen quad mesh. Geometry is aligned to the viewport corners.
     *
     * @author bmanuel
     * @author metaphore
     */
    private static class ViewportQuadMesh implements Disposable {

            private static final int VERT_SIZE = 16;
            private static final int X1 = 0;
            private static final int Y1 = 1;
            private static final int U1 = 2;
            private static final int V1 = 3;
            private static final int X2 = 4;
            private static final int Y2 = 5;
            private static final int U2 = 6;
            private static final int V2 = 7;
            private static final int X3 = 8;
            private static final int Y3 = 9;
            private static final int U3 = 10;
            private static final int V3 = 11;
            private static final int X4 = 12;
            private static final int Y4 = 13;
            private static final int U4 = 14;
            private static final int V4 = 15;

            private static final float[] verts;

            static {
                verts = new float[VERT_SIZE];

                // Vertex coords
                verts[X1] = -1f;
                verts[Y1] = -1f;
                verts[X2] = 1f;
                verts[Y2] = -1f;
                verts[X3] = 1f;
                verts[Y3] = 1f;
                verts[X4] = -1f;
                verts[Y4] = 1f;

                // Tex coords
                verts[U1] = 0f;
                verts[V1] = 0f;
                verts[U2] = 1f;
                verts[V2] = 0f;
                verts[U3] = 1f;
                verts[V3] = 1f;
                verts[U4] = 0f;
                verts[V4] = 1f;
            }

        private final Mesh mesh;

        public ViewportQuadMesh(VertexAttribute... attributes) {
            mesh = new Mesh(true, 4, 0, attributes);
            mesh.setVertices(verts);
        }

        @Override
        public void dispose() {
            mesh.dispose();
        }

        /** Renders the quad with the specified shader program. */
        public void render(ShaderProgram program) {
            mesh.render(program, GL20.GL_TRIANGLE_FAN, 0, 4);
        }
    }

    public static ShaderProgram compileShader(FileHandle vertexFile, FileHandle fragmentFile, String defines) {
        if (fragmentFile == null) {
            throw new IllegalArgumentException("Vertex shader file cannot be null.");
        }
        if (vertexFile == null) {
            throw new IllegalArgumentException("Fragment shader file cannot be null.");
        }
        if (defines == null) {
            throw new IllegalArgumentException("Defines cannot be null.");
        }

        StringBuilder sb = new StringBuilder();
        sb.append("Compiling \"").append(vertexFile.name()).append('/').append(fragmentFile.name()).append('\"');
        if (defines.length() > 0) {
            sb.append(" w/ (").append(defines.replace("\n", ", ")).append(")");
        }
        sb.append("...");
        Gdx.app.log("HqnxEffect", sb.toString());

        String srcVert = vertexFile.readString();
        String srcFrag = fragmentFile.readString();
        ShaderProgram shader = new ShaderProgram(defines + "\n" + srcVert, defines + "\n" + srcFrag);

        if (!shader.isCompiled()) {
            throw new GdxRuntimeException("Shader compilation error: " + vertexFile.name() + "/" + fragmentFile.name() + "\n" + shader.getLog());
        }
        return shader;
    }
}

HqnxEffect.kt

import com.badlogic.gdx.Gdx
import com.badlogic.gdx.files.FileHandle
import com.badlogic.gdx.graphics.*
import com.badlogic.gdx.graphics.VertexAttributes.Usage
import com.badlogic.gdx.graphics.glutils.FrameBuffer
import com.badlogic.gdx.graphics.glutils.ShaderProgram
import com.badlogic.gdx.utils.Disposable
import com.badlogic.gdx.utils.GdxRuntimeException

/**
 * [HQnX](https://en.wikipedia.org/wiki/Pixel-art_scaling_algorithms#hqnx_family)
 * upscale algorithm GLSL implementation based on
 * [CrossVR](https://github.com/CrossVR/hqx-shader/tree/master/glsl) project.
 */
class HqnxEffect : Disposable {

    companion object {
        private const val TEXTURE_HANDLE0 = 0
        private const val TEXTURE_HANDLE1 = 1

        private const val U_TEXTURE = "u_texture"
        private const val U_LUT = "u_lut"
        private const val U_TEXTURE_SIZE = "u_textureSize"
    }

    private val mesh = ViewportQuadMesh(
            VertexAttribute(Usage.Position, 2, "a_position"),
            VertexAttribute(Usage.TextureCoordinates, 2, "a_texCoord0"))

    private val program: ShaderProgram
    private val lutTexture: Texture
    private val scaleFactor: Int

    private var dstBuffer: FrameBuffer? = null
    private var dstWidth = 0
    private var dstHeight = 0

    /** @param scaleFactor should be 2, 3 or 4 value. */
    constructor(scaleFactor: Int) {
        if (scaleFactor !in 2..4) {
            throw GdxRuntimeException("Scale factor should be 2, 3 or 4.")
        }

        program = compileShader(
                Gdx.files.internal("hqnx.vert"),
                Gdx.files.internal("hqnx.frag"),
                "#define SCALE ${scaleFactor}.0")

        lutTexture = Texture(Gdx.files.internal("hq${scaleFactor}x.png"))

        this.scaleFactor = scaleFactor
    }

    override fun dispose() {
        mesh.dispose()
        program.dispose()
        lutTexture.dispose()
        dstBuffer?.dispose()
    }

    fun rebind() {
        program.bind()
        program.setUniformi(U_TEXTURE, TEXTURE_HANDLE0)
        program.setUniformi(U_LUT, TEXTURE_HANDLE1)
        program.setUniformf(U_TEXTURE_SIZE,
                dstWidth / scaleFactor.toFloat(),
                dstHeight / scaleFactor.toFloat())
    }

    fun renderToScreen(src: Texture) {
        validate(src)

        lutTexture.bind(TEXTURE_HANDLE1)
        src.bind(TEXTURE_HANDLE0)

        program.bind()
        mesh.render(program)
    }

    fun renderToBuffer(src: Texture): Texture {
        validate(src)
        validateDstBuffer()

        lutTexture.bind(TEXTURE_HANDLE1)
        src.bind(TEXTURE_HANDLE0)

        dstBuffer!!.begin()
        program.bind()
        mesh.render(program)
        dstBuffer!!.end()

        return dstBuffer!!.colorBufferTexture
    }

    private fun validate(src: Texture) {
        val targetWidth = src.width * scaleFactor
        val targetHeight = src.height * scaleFactor

        if (dstWidth != targetWidth || dstHeight != targetHeight) {
            dstWidth = targetWidth
            dstHeight = targetHeight
            rebind()
        }
    }

    private fun validateDstBuffer() {
        if (dstBuffer == null || dstBuffer!!.width != dstWidth || dstBuffer!!.height != dstHeight) {
            dstBuffer?.dispose()
            dstBuffer = FrameBuffer(Pixmap.Format.RGB888, dstWidth, dstHeight, false)
            dstBuffer!!.colorBufferTexture.setFilter(
                    Texture.TextureFilter.Nearest,
                    Texture.TextureFilter.Nearest)
        }
    }
}

/**
 * Encapsulates a fullscreen quad mesh. Geometry is aligned to the viewport corners.
 *
 * @author bmanuel
 * @author metaphore
 */
private class ViewportQuadMesh : Disposable {

    companion object {
        private const val VERT_SIZE = 16
        private const val X1 = 0
        private const val Y1 = 1
        private const val U1 = 2
        private const val V1 = 3
        private const val X2 = 4
        private const val Y2 = 5
        private const val U2 = 6
        private const val V2 = 7
        private const val X3 = 8
        private const val Y3 = 9
        private const val U3 = 10
        private const val V3 = 11
        private const val X4 = 12
        private const val Y4 = 13
        private const val U4 = 14
        private const val V4 = 15

        private val verts: FloatArray

        init {
            verts = FloatArray(VERT_SIZE)

            // Vertex coords
            verts[X1] = -1f
            verts[Y1] = -1f
            verts[X2] = 1f
            verts[Y2] = -1f
            verts[X3] = 1f
            verts[Y3] = 1f
            verts[X4] = -1f
            verts[Y4] = 1f

            // Tex coords
            verts[U1] = 0f
            verts[V1] = 0f
            verts[U2] = 1f
            verts[V2] = 0f
            verts[U3] = 1f
            verts[V3] = 1f
            verts[U4] = 0f
            verts[V4] = 1f
        }
    }

    private val mesh: Mesh

    constructor(vararg attributes: VertexAttribute) {
        mesh = Mesh(true, 4, 0, *attributes)
        mesh.setVertices(verts)
    }

    override fun dispose() {
        mesh.dispose()
    }

    /** Renders the quad with the specified shader program. */
    fun render(program: ShaderProgram) {
        mesh.render(program, GL20.GL_TRIANGLE_FAN, 0, 4)
    }
}

private fun compileShader(vertexFile: FileHandle, fragmentFile: FileHandle, defines: String): ShaderProgram {
    val sb = StringBuilder()
    sb.append("Compiling \"").append(vertexFile.name()).append('/').append(fragmentFile.name()).append('\"')
    if (defines.isNotEmpty()) {
        sb.append(" w/ (").append(defines.replace("\n", ", ")).append(")")
    }
    sb.append("...")
    Gdx.app.log("HqnxEffect", sb.toString())

    val srcVert = vertexFile.readString()
    val srcFrag = fragmentFile.readString()
    val shader = ShaderProgram(
            "$defines\n$srcVert".trimIndent(),
            "$defines\n$srcFrag".trimIndent())

    if (!shader.isCompiled) {
        throw GdxRuntimeException("Shader compilation error: ${vertexFile.name()}/${fragmentFile.name()}\n${shader.log}")
    }
    return shader
}

HQX

One file Java and Kotlin runtime effect implementation of the pixel art/line art upscale HQX algorithm.
It was mostly used to upscale old console pixel art graphics in the emulators, but still might be useful for visual effects or any other special processing at runtime.

The implementation is based on CrossVR project.

How to install

1. Copy HqnxEffect.java or HqnxEffect.kt (depends on the language preference) to your project.
2. Copy hqnx.vert and hqnx.frag GLSL shader code files into the root of your assets dir.
3. Download these images and add them to the root of your assets dir (GIST doesn't allow me to attach PNG files a usual way).
   HQnX LUT textures: hq2x hq3x hq4x.

If you prefer to have these resource files somewhere else in the assets hierarchy, just update the constructor code of the HqnxEffect class to load the files from a proper place:

...
program = compileShader(
                Gdx.files.internal("hqnx.vert"),                               // <---- update here
                Gdx.files.internal("hqnx.frag"),                               // <---- update here
                "#define SCALE " + scaleFactor + ".0");

lutTexture = new Texture(Gdx.files.internal("hq" + scaleFactor + "x.png"));    // <---- update here
...

How to use

HqnxEffect is a simple in-out processing unit. It takes a texture as input and renders an upscaled (2, 3 or 4 times) result directly to the screen
(#renderToScreen(Texture)) or outputs a texture that can be rendered separately or used for further processing (#renderToBuffer(Texture)).
For the latter method, it manages an internal FrameBuffer instance and resizes it based on the size of the input Texture parameter. So please keep in mind if you process textures with different sizes it will cause a lot of re-instantiation of the native resources related to FrameBuffer and thus you should consider having a HqnxEffect instance per one texture size, or adapt the internal implementation to your need.

Here's a basic pseudo-code of the general use case. It takes the captured game scene as an input texture and then renders the processed result to the screen.

HqnxEffect hqnxEffect;

void create() {
    // Create an effect instance with 2, 3 or 4 scale factor.
    hqnxEffect = new HqnxEffect(4);
}

void render() {
    //  Capture your game render into a FrameBuffer instance and pass the resulting texture as input to the HqnxEffect.
    Texture rawCapturedScene = ...
    HqnxEffect.renderToScreen(rawCapturedScene);
}

void dispose() {
    // The effect instance manages some internal resources, don't forget to dispose it.
    hqnxEffect.dispose();
}