Refs: https://github.com/Elysia-simp/Honse-Shader

UmaFaceShadow.shader

Shader "Custom/UmaFaceShadow" {
  Properties {
    _MainTex ("Diffuse Texture", 2D) = "white" {}
    _TripleMaskMap ("Triple Mask Map", 2D) = "white" {}
    _ToonStep ("Toon Step", Range(0, 1)) = 0.4
    _ToonFeather ("Toon Feather", Range(0.001, 1)) = 0.001
    _ShadowColor ("Shadow Color", Color) = (1, 0, 0, 1)
    _CheekPretenseThreshold ("Cheek Threshold", Range(0, 1)) = 0.775
    _NosePretenseThreshold ("Nose Threshold", Range(0, 1)) = 0.775
    _NoseVisibility ("Nose Visibility", Range(0, 1)) = 1.0
  }

  SubShader {
    Tags {
      "RenderType" = "Opaque"
      "RenderPipeline" = "UniversalPipeline"
      "Queue" = "Geometry"
    }

    Pass {
      Tags { "LightMode" = "UniversalForward" }

      HLSLPROGRAM
      #pragma vertex vert
      #pragma fragment frag

      #include "Packages/com.unity.render-pipelines.universal/ShaderLibrary/Core.hlsl"
      #include "Packages/com.unity.render-pipelines.universal/ShaderLibrary/Lighting.hlsl"

      struct Attributes {
        float4 positionOS : POSITION;
        float3 normalOS : NORMAL;
        float2 uv : TEXCOORD0;
      };

      struct Varyings {
        float4 positionCS : SV_POSITION;
        float2 uv : TEXCOORD0;
        float3 normalWS : TEXCOORD1;
        float3 positionWS : TEXCOORD2;
      };

      TEXTURE2D(_MainTex);
      SAMPLER(sampler_MainTex);
      TEXTURE2D(_TripleMaskMap);
      SAMPLER(sampler_TripleMaskMap);

      CBUFFER_START(UnityPerMaterial)
        float4 _MainTex_ST;
        float _ToonStep;
        float _ToonFeather;
        float3 _ShadowColor;
        float _CheekPretenseThreshold;
        float _NosePretenseThreshold;
        float _NoseVisibility;
      CBUFFER_END

      Varyings vert(Attributes input) {
        Varyings output;
        VertexPositionInputs vertexInput = GetVertexPositionInputs(input.positionOS.xyz);
        VertexNormalInputs normalInput = GetVertexNormalInputs(input.normalOS);

        output.positionCS = vertexInput.positionCS;
        output.positionWS = vertexInput.positionWS;
        output.uv = TRANSFORM_TEX(input.uv, _MainTex);
        output.normalWS = normalInput.normalWS;

        return output;
      }

      // Calculate toon shading factor based on UMA's original algorithm
      float CalculateToonFactor(float ndotl, float maskValue) {
        float toonThreshold = _ToonStep - _ToonFeather;
        toonThreshold = toonThreshold - (maskValue * ndotl);

        float toonStep = _ToonStep - (maskValue * ndotl);
        toonStep = saturate((toonStep - 0.05) * 20.0 + 1.0);

        float toonFactor = saturate(toonThreshold / _ToonFeather + 1.0);
        
        // Handle no feathering case
        if (_ToonFeather <= 0.0) {
          toonFactor = 0.0;
        }

        return toonFactor;
      }

      // Calculate face orientation factors for cheek/nose detection
      void CalculateFaceFactors(float3 lightDir, float3 faceForward, float3 faceUp, out float faceDotTemp, out float upDot, out float forwardDot) {
        // Face dot calculation with offset and clamping
        float faceDot = max(dot(faceForward, -lightDir) + 0.1, 0.0);
        faceDotTemp = max(-abs((-faceDot) + 0.5) * 2.0 + 1.0, 0.0);

        // Up and forward dot calculations
        upDot = min(dot(faceUp, lightDir), 0.0) + 1.0;
        forwardDot = saturate((-abs(dot(faceForward, lightDir)) + 1.0) * 2.0);
      }

      // Calculate blend factor for left/right face selection
      float CalculateBlendFactor(float3 lightDir, float3 faceForward, float3 faceUp, float3 localPos) {
        float3 lightCross = cross(faceForward, -lightDir);
        float lightSign = (dot(lightCross, faceUp) >= 0.0) ? 1.0 : -1.0;

        float3 posCross = cross(faceForward, localPos);
        float posSign = (dot(posCross, faceUp) >= 0.0) ? 1.0 : -1.0;

        return (posSign * lightSign + 1.0) * 0.5;
      }

      half4 frag(Varyings input) : SV_Target {
        float4 diffuse = SAMPLE_TEXTURE2D(_MainTex, sampler_MainTex, input.uv);

        // Triple Mask
        //  - Red: Shadow Area Definition. triple.x = 0.0: Dynamic shadowed area, triple.x = 1.0: Always shadowed area
        //  - Green: Cheek and Nose Area Definition. triple.y >= 0.51: Cheek area, triple.y <= 0.49: Nose area
        float4 triple = SAMPLE_TEXTURE2D(_TripleMaskMap, sampler_TripleMaskMap, input.uv);

        // Lighting setup
        Light light = GetMainLight();
        float3 lightDir = normalize(light.direction);
        float3 normal = normalize(input.normalWS);
        float ndotl = dot(normal, lightDir) * 0.5 + 0.5;

        // Calculate base toon shading
        float toonFactor = CalculateToonFactor(ndotl, triple.x);
        float4 color = lerp(diffuse, float4(_ShadowColor, 1.0), toonFactor);

        // Face coordinate system setup
        float3 faceForward = TransformObjectToWorldDir(float3(0, 0, 1));
        float3 faceUp = TransformObjectToWorldDir(float3(0, 1, 0));
        float3 faceCenterPos = GetObjectToWorldMatrix()._m03_m13_m23;
        float3 localPos = input.positionWS - faceCenterPos;

        // Calculate face orientation factors
        float faceDotTemp, upDot, forwardDot;
        CalculateFaceFactors(lightDir, faceForward, faceUp, faceDotTemp, upDot, forwardDot);

        // Calculate blend factor for face side selection
        float blendFactor = CalculateBlendFactor(lightDir, faceForward, faceUp, localPos);
        float3 blendedColor1 = lerp(_ShadowColor, diffuse.xyz, blendFactor);
        float3 blendedColor2 = lerp(diffuse.xyz, _ShadowColor, blendFactor);

        // Cheek shadow detection and application
        float cheekFactor = upDot * upDot * faceDotTemp;
        float tripleYOffset = max(triple.y - 0.51, 0.0);
        float toonStepNormalized = saturate((_ToonStep - triple.x * ndotl - 0.05) * 20.0 + 1.0);
        float cheekMask = toonStepNormalized * tripleYOffset;
        float cheekTest = dot(float2(cheekMask, cheekMask), float2(cheekFactor, cheekFactor));
        
        bool cheekActive = (cheekTest >= (1.0 - _CheekPretenseThreshold)) && (triple.y >= 0.51);
        color.xyz = lerp(color.xyz, blendedColor1, cheekActive ? 1.0 : 0.0);

        // Nose shadow detection and application
        float noseCalc = forwardDot * (min(triple.y, 0.49) * -2.0 + 1.0);
        bool noseActive = noseCalc >= (1.0 - _NosePretenseThreshold);
        bool noseVisible = triple.y <= 0.49;
        float noseMask = (noseActive && noseVisible) ? _NoseVisibility : 0.0;
        
        color.xyz = lerp(color.xyz, blendedColor2, noseMask);

        return color;
      }
      ENDHLSL
    }
  }
}