A "MeshStyle" for Starling that allows batching of multiple textures in one draw call.

MultiTextureStyle.as

// =================================================================================================
//
//	Starling Framework
//	Copyright Gamua GmbH. All Rights Reserved.
//
//	This program is free software. You can redistribute and/or modify it
//	in accordance with the terms of the accompanying license agreement.
//
// =================================================================================================

package starling.extensions
{
    import flash.geom.Matrix;

    import starling.display.Mesh;
    import starling.rendering.MeshEffect;
    import starling.rendering.RenderState;
    import starling.rendering.VertexData;
    import starling.rendering.VertexDataFormat;
    import starling.styles.MeshStyle;
    import starling.textures.Texture;

    public class MultiTextureStyle extends MeshStyle
    {
        public static const VERTEX_FORMAT:VertexDataFormat =
            MeshStyle.VERTEX_FORMAT.extend("texture:float1");

        public static const MAX_NUM_TEXTURES:int = 4;

        private var _textures:Vector.<Texture>;

        private static var sTextureIndexMap:Array = [];

        public function MultiTextureStyle()
        {
            _textures = new <Texture>[];
        }

        override public function createEffect():MeshEffect
        {
            return new MultiTextureEffect();
        }

        override public function updateEffect(effect:MeshEffect, state:RenderState):void
        {
            var targetEffect:MultiTextureEffect = effect as MultiTextureEffect;
            var numTextures:int = _textures.length;

            targetEffect.clearTextures();

            super.updateEffect(effect, state);

            for (var i:int=0; i<numTextures; ++i)
                targetEffect.setTextureAt(i, _textures[i]);
        }

        override public function canBatchWith(meshStyle:MeshStyle):Boolean
        {
            var mtStyle:MultiTextureStyle = meshStyle as MultiTextureStyle;
            if (mtStyle)
            {
                var i:int;
                var numTexturesToAdd:int = _textures.length;

                if (mtStyle.numTextures + numTexturesToAdd > MAX_NUM_TEXTURES)
                {
                    var numSharedTextures:int = 0;

                    for (i=0; i<numTexturesToAdd; ++i)
                        if (mtStyle.getTextureIndex(_textures[i]) != -1)
                            numSharedTextures++;

                    return mtStyle.numTextures + numTexturesToAdd - numSharedTextures
                        <= MAX_NUM_TEXTURES;
                }
                else return true;
            }

            return false;
        }

        override public function batchVertexData(target:MeshStyle, targetVertexID:int = 0,
                                                 matrix:Matrix = null, vertexID:int = 0,
                                                 numVertices:int = -1):void
        {
            super.batchVertexData(target, targetVertexID, matrix, vertexID, numVertices);

            var mtTarget:MultiTextureStyle = target as MultiTextureStyle;
            if (mtTarget)
            {
                var targetVertexData:VertexData = mtTarget.vertexData;
                var sourceVertexData:VertexData = this.vertexData;
                var numTextures:int = _textures.length;
                var sourceTexID:int, targetTexID:int;
                var i:int;

                if (numVertices < 0)
                    numVertices = vertexData.numVertices - vertexID;

                if (targetVertexID == 0)
                    mtTarget._textures.length = 0;

                for (i = 0; i < numTextures; ++i)
                {
                    var texture:Texture = _textures[i];
                    var textureIndexOnTarget:int = mtTarget.getTextureIndex(texture);

                    if (textureIndexOnTarget == -1)
                    {
                        textureIndexOnTarget = mtTarget.numTextures;
                        mtTarget.setTextureAt(texture, textureIndexOnTarget);
                    }

                    sTextureIndexMap[i] = textureIndexOnTarget;
                }

                for (i = 0; i < numVertices; ++i)
                {
                    // TODO
                    // we could make this more efficient by storing the regions in which certain
                    // textures are used in a separate data structure, instead of accessing the
                    // vertex data over and over.

                    if (numTextures == 0) sourceTexID = -1;
                    else sourceTexID = sourceVertexData.getFloat(vertexID + i, "texture");

                    if (sourceTexID == -1) targetTexID = -1;
                    else targetTexID = sTextureIndexMap[sourceTexID];

                    if (sourceTexID == -1 || sourceTexID != targetTexID)
                        targetVertexData.setFloat(targetVertexID + i, "texture", targetTexID);
                }

                sTextureIndexMap.length = 0;
            }
        }

        override protected function onTargetAssigned(target:Mesh):void
        {
            _textures.length = 0;
            if (target.texture) _textures[0] = target.texture;
        }

        override public function get vertexFormat():VertexDataFormat
        {
            return VERTEX_FORMAT;
        }

        override public function set texture(value:Texture):void
        {
            if (value) _textures[0] = value;
            super.texture = value;
        }

        private function setTextureAt(texture:Texture, index:int):void
        {
            _textures[index] = texture;
        }

        private function getTextureIndex(texture:Texture):int
        {
            var numTextures:int = _textures.length;

            for (var i:int=0; i<numTextures; ++i)
                if (_textures[i].base == texture.base) return i;

            return -1;
        }

        private function get numTextures():int { return _textures.length; }
    }
}

import flash.display3D.Context3D;
import flash.display3D.Context3DProgramType;

import starling.core.Starling;
import starling.rendering.MeshEffect;
import starling.rendering.Program;
import starling.rendering.VertexDataFormat;
import starling.styles.MultiTextureStyle;
import starling.textures.Texture;
import starling.utils.RenderUtil;

class MultiTextureEffect extends MeshEffect
{
    private var _textures:Vector.<Texture>;

    private static const sTextureIndices:Vector.<Number> = new <Number>[0, 1, 2, 3];
    private static const sOnes:Vector.<Number> = new <Number>[1, 1, 1, 1];

    public function MultiTextureEffect()
    {
        _textures = new <Texture>[];
    }

    override protected function createProgram():Program
    {
        var vertexShader:Array = [
            "m44 op, va0, vc0", // 4x4 matrix transform to output clip-space
            "mov v0, va1     ", // pass texture coordinates to fragment program
            "mul v1, va2, vc4", // multiply alpha (vc4) with color (va2), pass to fp
            "mov v2, va3     "  // pass texture sampler index to fp
        ];

        // fc0 = [0, 1, 2, 3]
        // fc1 = [1, 1, 1, 1]

        var isBaseline:Boolean = Starling.current.profile.indexOf("baseline") != -1;
        var agalVersion:uint = isBaseline ? 1 : 2;
        var fragmentShader:Array = isBaseline ?
            createFragmentShaderForBaselineProfile(numTextures) :
            createFragmentShaderForStandardProfile(numTextures);

        return Program.fromSource(vertexShader.join("\n"), fragmentShader.join("\n"), agalVersion);
    }

    private function createFragmentShaderForBaselineProfile(numTextures:int):Array
    {
        // In baseline profile, if-statements are not available. Instead,
        // we sample all textures and multiply all but the active one with "zero".

        var fragmentShader:Array = [];

        if (numTextures == 0)
            fragmentShader.push("mov ft4, fc0.xxxx"); // init with zero

        if (numTextures > 0)
            fragmentShader.push(
                tex("ft0", "v0", 0, _textures[0]),
                "seq ft1, v2.x, fc0.x",
                "mul ft4, ft1, ft0"
            );

        if (numTextures > 1)
            fragmentShader.push(
                tex("ft0", "v0", 1, _textures[1]),
                "seq ft1, v2.x, fc0.y",
                "mul ft0, ft1, ft0",
                "add ft4, ft4, ft0"
            );

        if (numTextures > 2)
            fragmentShader.push(
                tex("ft0", "v0", 2, _textures[2]),
                "seq ft1, v2.x, fc0.z",
                "mul ft0, ft1, ft0",
                "add ft4, ft4, ft0"
            );

        if (numTextures > 3)
            fragmentShader.push(
                tex("ft0", "v0", 3, _textures[3]),
                "seq ft1, v2.x, fc0.w",
                "mul ft0, ft1, ft0",
                "add ft4, ft4, ft0"
            );

        fragmentShader.push(
            "slt ft0, v2.x, fc0.x", // no texture => v2.x < 0
            "add ft4, ft4, ft0",
            "mul oc, ft4, v1"       // multiply color with texel color
        );

        return fragmentShader;
    }

    private function createFragmentShaderForStandardProfile(numTextures:int):Array
    {
        // In standard profile, we can actually choose the correct texture via an if-operation,
        // which should be more efficient (fewer texture look-ups).

        var fragmentShader:Array = [
            "mov ft0, fc1" // init with white (used when no texture is assigned)
        ];

        if (numTextures > 0)
            fragmentShader.push(
                "ife v2.x, fc0.x",
                tex("ft0", "v0", 0, _textures[0]),
                "eif"
            );

        if (numTextures > 1)
            fragmentShader.push(
                "ife v2.x, fc0.y",
                tex("ft0", "v0", 1, _textures[1]),
                "eif"
            );

        if (numTextures > 2)
            fragmentShader.push(
                "ife v2.x, fc0.z",
                tex("ft0", "v0", 2, _textures[2]),
                "eif"
            );

        if (numTextures > 3)
            fragmentShader.push(
                "ife v2.x, fc0.w",
                tex("ft0", "v0", 3, _textures[3]),
                "eif"
            );

        fragmentShader.push("mul oc, ft0, v1"); // multiply color with texel color
        return fragmentShader;
    }

    override protected function beforeDraw(context:Context3D):void
    {
        super.beforeDraw(context);

        context.setProgramConstantsFromVector(Context3DProgramType.FRAGMENT, 0, sTextureIndices);
        context.setProgramConstantsFromVector(Context3DProgramType.FRAGMENT, 1, sOnes);
        vertexFormat.setVertexBufferAt(1, vertexBuffer, "texCoords");
        vertexFormat.setVertexBufferAt(3, vertexBuffer, "texture");

        for (var i:int=0; i<numTextures; ++i)
        {
            var texture:Texture = _textures[i];
            RenderUtil.setSamplerStateAt(i, texture.mipMapping, textureSmoothing, textureRepeat);
            context.setTextureAt(i, texture.base);
        }
    }

    override protected function afterDraw(context:Context3D):void
    {
        for (var i:int=0; i<numTextures; ++i)
            context.setTextureAt(i, null);

        context.setVertexBufferAt(1, null);
        context.setVertexBufferAt(3, null);

        super.afterDraw(context);
    }

    override public function get vertexFormat():VertexDataFormat
    {
        return MultiTextureStyle.VERTEX_FORMAT;
    }

    override protected function get programVariantName():uint
    {
        var numTextures:int = _textures.length;
        var bits:uint = 0;

        for (var i:int=0; i<numTextures; ++i)
            bits |= RenderUtil.getTextureVariantBits(_textures[i]) << (4 * i);

        return bits;
    }

    override public function set texture(value:Texture):void
    {
        if (value) _textures[0] = value;
        super.texture = value;
    }

    public function setTextureAt(index:int, texture:Texture):void
    {
        _textures[index] = texture;
    }

    public function clearTextures():void
    {
        _textures.length = 0;
    }

    public function get numTextures():int { return _textures.length; }
}

@Fancy2209 I've added a fork to this with a version we've been using successfully for a while now. Hope it works for you.

I actually think I found an issue
if I use the original and set the Max Texture to 2, then render 4 Textures, I get 2 draw calls like expected but something gets rendered
With your fork i just get a white screen unless I change the maxTextures to 4, so it's batched in 1 call

@Fancy2209 I think I found the bug and fixed it. It only occurred in baseline mode, which we never use, so I hadn't noticed it before. I've updated the fork.

@Fancy2209 I think I found the bug and fixed it. It only occurred in baseline mode, which we never use, so I hadn't noticed it before. I've updated the fork.

Ah yep, I was using Linux flash player who is always on a software context, that explains it, thanks!

@Fancy2209 Actually, I never got around to fixing this, I'm afraid! I can't remember why, as I still think it's actually a great idea to save draw calls.

Thus, I'm super happy to see that your fork did the trick, @johnridges! If that planned new release version ever comes around, would you be okay with me including this? I probably wouldn't make it the default, but that way anyone could doe so with a simple one-liner.