Explains the issue with a separate stencil test

stencil-test-separate.elm

module Main exposing (..)

import Math.Vector3 exposing (..)
import WebGL exposing (Shader, Mesh)
import WebGL.Settings exposing (Setting)
import WebGL.Settings.StencilTest as StencilTest
import Html exposing (Html)
import Html.Attributes exposing (width, height)


{-| Draws a red and a green triangles, where the green triangle is only
visible in the intercection with the red triangle.

A green outline marks a hidden area of the green triangle.
-}
main : Html ()
main =
    WebGL.toHtmlWith
        [ WebGL.stencil 0 ]
        [ width 400, height 400 ]
        [ WebGL.entityWith
            [ stencilTest ]
            vertexShader
            fragmentShader
            redAndGreenTriangles
            {}
        , WebGL.entity
            vertexShader
            fragmentShader
            greenOutline
            {}
        ]


{-| Rendering with the following setting will write 1's to the stencil buffer
for all front-facing triangles, and then test the subsequent back-facing
triangles against the stencil buffer, so they be rendered only for the areas
that are marked with 1's in the stencil buffer.

`StencilTest.testSeparate` takes two options, one for front-, and another for
back-facing triangles:

* The front-facing stencil test always passes, and replaces the stencil buffer
  with 1.
* The back-facing stencil test only passes when the value in the stencil buffer
  is equal to 1. It does not modify the stencil buffer.

Due to [WebGL limitations](https://www.khronos.org/registry/webgl/specs/latest/1.0/#6.10),
both tests should use the same `ref`, `mask` and `writeMask`. Changing one of
these will result in the following error: "WebGL: INVALID_OPERATION:
drawElements: front and back stencils settings do not match".
-}
stencilTest : Setting
stencilTest =
    let
        ref =
            1

        mask =
            0xFF

        writeMask =
            0xFF
    in
        StencilTest.testSeparate
            { test = StencilTest.always ref mask
            , fail = StencilTest.keep
            , zfail = StencilTest.keep
            , zpass = StencilTest.replace
            , writeMask = writeMask
            }
            { test = StencilTest.equal ref mask
            , fail = StencilTest.keep
            , zfail = StencilTest.keep
            , zpass = StencilTest.keep
            , writeMask = writeMask
            }



-- Mesh


type alias Vertex =
    { position : Vec3
    , color : Vec3
    }


redAndGreenTriangles : Mesh Vertex
redAndGreenTriangles =
    let
        -- the red triangle is front-facing
        redTriangle =
            ( Vertex (vec3 -1 0.5 0) (vec3 1 0 0)
            , Vertex (vec3 0 -0.5 0) (vec3 1 0 0)
            , Vertex (vec3 1 0.5 0) (vec3 1 0 0)
            )

        -- the green triangle is back-facing
        greenTriangle =
            ( Vertex (vec3 -1 -0.5 0) (vec3 0 1 0)
            , Vertex (vec3 0 0.5 0) (vec3 0 1 0)
            , Vertex (vec3 1 -0.5 0) (vec3 0 1 0)
            )
    in
        WebGL.triangles
            [ redTriangle
            , greenTriangle
            ]


greenOutline : Mesh Vertex
greenOutline =
    WebGL.lineLoop
        [ Vertex (vec3 -1 -0.5 0) (vec3 0 1 0)
        , Vertex (vec3 0 0.5 0) (vec3 0 1 0)
        , Vertex (vec3 1 -0.5 0) (vec3 0 1 0)
        ]



-- Shaders


vertexShader : Shader Vertex {} { vcolor : Vec3 }
vertexShader =
    [glsl|

        attribute vec3 position;
        attribute vec3 color;
        varying vec3 vcolor;

        void main () {
            gl_Position = vec4(position, 1.0);
            vcolor = color;
        }

    |]


fragmentShader : Shader {} {} { vcolor : Vec3 }
fragmentShader =
    [glsl|

        precision mediump float;
        varying vec3 vcolor;

        void main () {
            gl_FragColor = vec4(vcolor, 1.0);
        }

    |]

suggestions.elm

-- the current thing

settingSeparate = StencilTest.testSeparate
    { test = StencilTest.always 1 0xFF
    , fail = StencilTest.keep
    , zfail = StencilTest.keep
    , zpass = StencilTest.replace
    , writeMask = 0xFF
    }
    { test = StencilTest.equal 1 0xFF
    , fail = StencilTest.keep
    , zfail = StencilTest.keep
    , zpass = StencilTest.keep
    , writeMask = 0xFF
    }

-- Should Become:

settingSeparate = StencilTest.testSeparate
    { ref = 1
    , mask = 0xFF
    , writeMask = 0xFF
    }
    { test = StencilTest.always
    , fail = StencilTest.keep
    , zfail = StencilTest.keep
    , zpass = StencilTest.replace
    }
    { test = StencilTest.equal
    , fail = StencilTest.keep
    , zfail = StencilTest.keep
    , zpass = StencilTest.keep
    }
    
    
-- Or maybe group the tests and operations:

settingSeparate = StencilTest.performSeparate
    { ref = 1
    , mask = 0xFF
    , writeMask = 0xFF
    }
    (StencilTest.always StencilTest.keep StencilTest.keep StencilTest.replace)
    (StencilTest.equal StencilTest.keep StencilTest.keep StencilTest.keep)

-- The single test will be

setting = StencilTest.perform
    { ref = 1
    , mask = 0xFF
    , writeMask = 0xFF
    }
    (StencilTest.always StencilTest.keep StencilTest.keep StencilTest.replace)
    
-- Or if we initiate test with all operations set to keep, and provide mapFail, mapZFail, mapZPass

settingSeparate = StencilTest.performSeparate
    { ref = 1
    , mask = 0xFF
    , writeMask = 0xFF
    }
    (StencilTest.mapZPass StencilTest.replace StencilTest.always)
    StencilTest.equal


setting = StencilTest.perform
    { ref = 1
    , mask = 0xFF
    , writeMask = 0xFF
    }
    (StencilTest.mapZPass StencilTest.always StencilTest.replace)

I think I like the explicitness of the first (all-record) approach. Also, pretty minor but I'd be tempted to go with zPass/zFail instead of zpass/zfail, especially if there's already mapZPass (otherwise for consistency it seems that should be mapZpass).

@ianmackenzie cool, I've decided to have all-record approach. mapZPass was from a different alternative, that we are not going to use.