Pixel perfect dithering in StarfallEx in realtime using stencils

cl_init.lua

--@name Chop's Rampage
--@client
--@include chopsrampage/dither.lua
dofile('chopsrampage/dither.lua')

render.createRenderTarget('display')

local testang = Angle()
local test = holograms.create(Vector(100, 50, -30), testang, 'models/starfall/holograms/box.mdl', Vector(150, 150, 150))
test:setNoDraw(true)
test:suppressEngineLighting(true)

local coverpos = Vector(
	40, -- please intersect
--	-100, -- actually don't
	0, 0
)
local coverang = Angle()
local cover = holograms.create(coverpos, coverang, 'models/starfall/holograms/hollowcylinder.mdl', Vector(50, 50, 50))
cover:setNoDraw(true)
cover:suppressEngineLighting(true)

render.createRenderTarget('texture1')
local mat1 = material.create('VertexLitGeneric')

render.createRenderTarget('texture2')
local mat2 = material.create('VertexLitGeneric')

render.createRenderTarget('dither')

hook.add('renderoffscreen', 'init', function()
	render.selectRenderTarget('texture1')
	render.clear(Color(255, 127, 127))
	render.setRGBA(127, 255, 127, 255)
	render.drawRect(512, 0, 512, 1024)
	mat1:setTextureRenderTarget('$basetexture', 'texture1')
	test:setMaterial('!'..mat1:getName())
	
	render.selectRenderTarget('texture2')
	render.clear(Color(127, 127, 255))
	mat2:setTextureRenderTarget('$basetexture', 'texture2')
	cover:setMaterial('!'..mat2:getName())
	
	createDitherPattern('dither', 256)
	
	hook.remove('renderoffscreen', 'init')
end)

-- why the odd res and aspect? it's so it looks like it's a SNES game.
local vmtx = {
	w = 256,
	h = 224,
	type = '3D',
	origin = Vector(-200, -50, 50),
	angles = Angle(10, 20, 0),
	fov = 70,
	aspect = 4/3
}

local DO_DITHER = true
local function drawWithDither(mesh, r, g, b, a)
	if not DO_DITHER then
		render.pushViewMatrix(vmtx)
			mesh:draw()
		render.popViewMatrix()
		return
	end
	render.resetStencil()
	render.clearStencil()
	render.setStencilEnable(true)
	render.setStencilWriteMask(0xff)
	render.setStencilTestMask(0xff)
	render.setStencilReferenceValue(0x01)
	render.setStencilCompareFunction(STENCIL.ALWAYS)
	render.setStencilPassOperation(STENCIL.REPLACE)
	render.setStencilFailOperation(STENCIL.REPLACE)
	render.setStencilZFailOperation(STENCIL.KEEP)
	render.pushViewMatrix(vmtx)
		render.setRGBA(255, 0, 0, 0)
		mesh:draw()
	render.popViewMatrix()
	render.setStencilCompareFunction(STENCIL.EQUAL)
	render.setRenderTargetTexture('dither')
	render.setRGBA(r, g, b, a)
	render.drawTexturedRect(0, 0, 1024, 1024)
	render.setStencilEnable(false)
end

local bg1 = Color(0, 0, 0)
local bg2 = Color(31, 31, 31)
hook.add('render', '', function()
	render.setBackgroundColor(bg1)
	render.setFilterMag(1)
	render.setRGBA(255, 255, 255, 255)
	
	render.selectRenderTarget('display')
		render.clear(bg2, true)
		
		render.pushViewMatrix(vmtx)
			-- [[
			coverpos.y = math.sin(timer.systime()*2)*40
			coverpos.z = math.sin(timer.systime()*2.35)*25
			coverang.p = math.sin(timer.systime()*2.95)*90
			coverang.y = math.sin(timer.systime()*3.15)*90
			coverang.r = math.sin(timer.systime()*3.35)*90
			cover:setPos(coverpos)
			cover:setAngles(coverang)
			--cover:draw()
			--]]
		render.popViewMatrix()
		
		testang.p = math.cos(timer.systime()*0.95)*90
		testang.y = math.cos(timer.systime()*1.15)*90
		testang.r = math.cos(timer.systime()*1.35)*90
		test:setAngles(testang)
		drawWithDither(test, 255, 255, 255, 63)
		
		-- [[
		render.pushViewMatrix(vmtx)
			cover:draw()
		render.popViewMatrix()
		--]]
		
		--render.setRGBA(255, 255, 255, 127)
		--render.setMaterial(refimg)
		--render.drawTexturedRect(0, 0, 1024, 1024)
	render.selectRenderTarget()
	
	render.setRGBA(255, 255, 255, 255)
	render.setRenderTargetTexture('display')
	render.drawTexturedRectUV(0, 64, 512, 384, 0, 0, 0.25, 0.21875)
	--render.drawTexturedRect(0, 0, 512, 512)
end)

dither.lua

local key = '_dither'
local transparent = Color(0, 0, 0, 0)
function createDitherPattern(rt, res, callback)
	assert(bit.band(res, res-1) == 0, "res must be a power of 2")
	assert(res > 2, "res cannot be less than 2")
	assert(res <= 1024, "res cannot be greater than 1024")
	local function init()
		render.selectRenderTarget(rt)
		render.clear(transparent, false)
		render.drawRectFast(0, 0, 1, 1)
		render.drawRectFast(1, 1, 1, 1)
		local pos = 2
		--local uv = pos/1024
		local uv = 0.001953125
		local function draw()
			render.selectRenderTarget(rt)
			render.setRenderTargetTexture(rt)
			render.drawTexturedRectUV(pos, 0, pos, pos, 0, 0, uv, uv)
			render.drawTexturedRectUV(pos, pos, pos, pos, 0, 0, uv, uv)
			render.drawTexturedRectUV(0, pos, pos, pos, 0, 0, uv, uv)
			pos = pos*2
			uv = uv*2
			if pos >= res then
				--hook.remove('render', key)
				hook.remove('renderoffscreen', key)
				if callback then
					callback(rt, res)
				end
			end
		end
		--hook.add('render', key, draw)
		hook.add('renderoffscreen', key, draw)
	end
	--hook.add('render', key, init)
	hook.add('renderoffscreen', key, init)
end
return createDitherPattern