scaleforce.frag

//!DESC ScaleForce
//!HOOK MAINPRESUB
//!BIND HOOKED
//!WIDTH OUTPUT.w
//!HEIGHT OUTPUT.h
//? #version 450 core

#ifndef HOOKED_tex
// Copy MPV built-ins for IDE
#define tex1D texture
#define tex3D texture
#define LUT_POS(x, lut_size) mix(0.5 / (lut_size), 1.0 - 0.5 / (lut_size), (x))
layout(location=0) out vec4 out_color;
layout(location=1) in vec2 texcoord0;
layout(std140, binding=0) uniform UBO {
    layout(offset=0) float random;
    layout(offset=4) int frame;
    layout(offset=8) vec2 input_size;
    layout(offset=16) vec2 target_size;
    layout(offset=24) vec2 tex_offset;
    layout(offset=32) vec2 texture_size0;
    layout(offset=48) mat2 texture_rot0;
    layout(offset=80) vec2 texture_off0;
    layout(offset=88) vec2 pixel_size0;
};
layout(binding=0) uniform sampler2D texture0;
#define HOOKED_raw texture0
#define HOOKED_pos texcoord0
#define HOOKED_size texture_size0
#define HOOKED_rot texture_rot0
#define HOOKED_off texture_off0
#define HOOKED_pt pixel_size0
#define HOOKED_map texmap0
#define HOOKED_mul 1.000000
#define HOOKED_tex(pos) (HOOKED_mul * vec4(texture(HOOKED_raw, pos)).rgba)
#define HOOKED_texOff(off) HOOKED_tex(HOOKED_pos + HOOKED_pt * vec2(off))

#endif

// MIT License
//
// Copyright (c) 2020 BreadFish64
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// Adapted from https://github.com/BreadFish64/ScaleFish/tree/master/scaleforce

#extension GL_ARB_separate_shader_objects : enable

#ifdef GL_AMD_gpu_shader_half_float
#extension GL_AMD_gpu_shader_half_float : enable
#define USE_FP16 1
#endif
#ifdef GL_NV_gpu_shader5
#extension GL_NV_gpu_shader5 : enable
#define USE_FP16 1
#endif

#if USE_FP16

#define lfloat float16_t
#define lvec2 f16vec2
#define lvec3 f16vec3
#define lvec4 f16vec4

#else

#define lfloat float
#define lvec2 vec2
#define lvec3 vec3
#define lvec4 vec4

#endif

const bool ignore_alpha = true;

lfloat ColorDist1(lvec4 a, lvec4 b) {
    // https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.2020_conversion
    const lvec3 K = lvec3(0.2627, 0.6780, 0.0593);
    const lfloat scaleB = lfloat(0.5) / (lfloat(1.0) - K.b);
    const lfloat scaleR = lfloat(0.5) / (lfloat(1.0) - K.r);
    lvec4 diff = a - b;
    lfloat Y = dot(diff.rgb, K);
    lfloat Cb = scaleB * (diff.b - Y);
    lfloat Cr = scaleR * (diff.r - Y);
    lvec3 YCbCr = lvec3(Y, Cb, Cr);
    lfloat d = length(YCbCr);
    if (ignore_alpha) {
        return d;
    }
    return sqrt(a.a * b.a * d * d + diff.a * diff.a);
}

lvec4 ColorDist(lvec4 ref, lvec4 A, lvec4 B, lvec4 C, lvec4 D) {
    return lvec4(
            ColorDist1(ref, A),
            ColorDist1(ref, B),
            ColorDist1(ref, C),
            ColorDist1(ref, D)
        );
}

vec4 Scaleforce(sampler2D tex, vec2 tex_coord) {
    lvec4 bl = lvec4(textureOffset(tex, tex_coord, ivec2(-1, -1)));
    lvec4 bc = lvec4(textureOffset(tex, tex_coord, ivec2(0, -1)));
    lvec4 br = lvec4(textureOffset(tex, tex_coord, ivec2(1, -1)));
    lvec4 cl = lvec4(textureOffset(tex, tex_coord, ivec2(-1, 0)));
    lvec4 cc = lvec4(texture(tex, tex_coord));
    lvec4 cr = lvec4(textureOffset(tex, tex_coord, ivec2(1, 0)));
    lvec4 tl = lvec4(textureOffset(tex, tex_coord, ivec2(-1, 1)));
    lvec4 tc = lvec4(textureOffset(tex, tex_coord, ivec2(0, 1)));
    lvec4 tr = lvec4(textureOffset(tex, tex_coord, ivec2(1, 1)));

    lvec4 offset_tl = ColorDist(cc, tl, tc, tr, cr);
    lvec4 offset_br = ColorDist(cc, br, bc, bl, cl);

    // Calculate how different cc is from the texels around it
    const lfloat plus_weight = lfloat(1.5);
    const lfloat cross_weight = lfloat(1.5);
    lfloat total_dist = dot(offset_tl + offset_br, lvec4(cross_weight, plus_weight, cross_weight, plus_weight));

    if (total_dist == lfloat(0.0)) {
        return cc;
    } else {
        // Add together all the distances with direction taken into account
        lvec4 tmp = offset_tl - offset_br;
        lvec2 total_offset = tmp.wy * plus_weight + (tmp.zz + lvec2(-tmp.x, tmp.x)) * cross_weight;

        // When the image has thin points, they tend to split apart.
        // This is because the texels all around are different and total_offset reaches into clear areas.
        // This works pretty well to keep the offset in bounds for these cases.
        lfloat clamp_val = length(total_offset) / total_dist;
        vec2 final_offset = vec2(clamp(total_offset, -clamp_val, clamp_val)) / textureSize(tex, 0);

        return texture(tex, tex_coord - final_offset);
    }
}

vec4 hook() {
    return Scaleforce(HOOKED_raw, HOOKED_pos);
}