PlayState.hx

package;

import flixel.FlxCamera;
import flixel.FlxG;
import flixel.FlxSprite;
import flixel.FlxState;
import flixel.util.FlxColor;
import openfl.display.BitmapData;
import openfl.filters.ShaderFilter;

class PlayState extends FlxState
{
	private var secondCamera:FlxCamera;
	private var thirdCamera:FlxCamera;
	private var shader:TransparentShader;
	private var mask:FlxSprite;
	private var shaderMaskSprite:FlxSprite;

	override public function create()
	{
		super.create();

		// cameras
		this.secondCamera = new FlxCamera();
		this.secondCamera.bgColor = FlxColor.TRANSPARENT;
		this.thirdCamera = new FlxCamera();
		thirdCamera.bgColor = FlxColor.TRANSPARENT;
		thirdCamera.x = 10000;

		FlxG.cameras.add(this.secondCamera);
		FlxG.cameras.add(this.thirdCamera);

		FlxCamera.defaultCameras = [FlxG.camera];

		// sprites
		var background = new FlxSprite();
		background.loadGraphic('assets/images/background4.png');

		var foreground = new FlxSprite();
		foreground.loadGraphic('assets/images/foreground5.png');
		foreground.camera = this.secondCamera;

		mask = new FlxSprite(60, 60);
		mask.loadGraphic('assets/images/mask3.png');
		mask.camera = this.thirdCamera;
		add(background);
		add(foreground);
		add(mask);

		// Add shader
		this.shader = new TransparentShader();
		this.secondCamera.setFilters([new ShaderFilter(this.shader)]);

		this.shaderMaskSprite = new FlxSprite();
		this.shaderMaskSprite.makeGraphic(FlxG.width, FlxG.height, FlxColor.TRANSPARENT, true);
	}

	override public function update(dt:Float)
	{
		super.update(dt);
		// Draw the canvas to the shader mask sprite
		this.shaderMaskSprite.pixels = new BitmapData(FlxG.width, FlxG.height, true, FlxColor.TRANSPARENT);
		this.shaderMaskSprite.pixels.draw(this.thirdCamera.canvas);
		// Send the shader mask sprite pixels to the shader.
		// I do this because I can't send the camera's bitmapdata to the shader directly.
		this.shader.tex.input = this.shaderMaskSprite.pixels;
	}
}

TransparentShader.hx

import flixel.system.FlxAssets.FlxShader;

class TransparentShader extends FlxShader
{
	@:glFragmentSource('
        #pragma header
        uniform sampler2D tex;
		void main()
		{
            vec4 source = texture2D(bitmap, openfl_TextureCoordv);
            vec4 mask = texture2D(tex, openfl_TextureCoordv);
            source.a *= (1.0 - mask.a);
            gl_FragColor = source;
        }
    ')
	public function new()
	{
		super();
	}
}