Houdini Solaris shader/render debug "isolate selected" type of behavior similar to Arnold Render View (use as shelf script - CTRL+ click disables it)

houdini_solaris_isolate_selected_debug_shading_shelf_button.py

from typing import Tuple

import hou
import toolutils
from pxr import UsdShade, Sdf

POST_LAYER_NAME = "isolate_shaders"


# Prioritize outputs if there are multiple
SHADER_OUTPUT_ORDER = [
    "outputs:surface",  # usual surface outputs
    "outputs:out",  # usual materialx output
    "outputs:shader",  # usual arnold outputs
    "outputs:rgb",
    "outputs:rgba",
    "outputs:vector",
    "outputs:float",
    "outputs:boolean",
    "outputs:out_variable",  # arnold 'state' outputs
    # Worst case scenario fall back to r, g or b channel
    "outputs:r",
    "outputs:g",
    "outputs:b"
]
SHADER_OUTPUT_ORDER = {key: i for i, key in enumerate(SHADER_OUTPUT_ORDER)}

BLACK_SHADER = """#sdf 1.0
def Material "__debug_black_mat"
{
    token outputs:surface.connect = </__debug_black_mat/usdpreviewsurface.outputs:surface>

    def Shader "usdpreviewsurface"
    {
        uniform token info:id = "UsdPreviewSurface"
        color3f inputs:diffuseColor = (0, 0, 0)
        float inputs:ior = 1
        float inputs:roughness = 0
        token outputs:surface
    }
}
"""


def get_main_output(shader):
    """Define what will be the output to display from a Shader

    This should basically mimic what the visualize node would do.
    """
    # TODO: Improve this logic so that if e.g. a R channel would
    #   be connected in the graph that it'd still show RGB if the
    #   shader id does support that output.
    outputs = shader.GetOutputs()
    if len(outputs) == 1:
        return outputs[0]

    if len(outputs) == 0:
        return

    def sorter(output):
        name = output.GetFullName()

        index = SHADER_OUTPUT_ORDER.get(name)
        if index is None:
            index = len(SHADER_OUTPUT_ORDER) + 1

        return index, name

    outputs.sort(key=sorter)
    return outputs[0]


def get_parent_material(shader):
    """From a UsdShade.Shader find the first parent that has a type name.

    This should usually return the "UsdShade.Material"
    """
    parent = shader.GetParent()
    while parent and not parent.GetTypeName():
        if parent.IsPseudoRoot():
            return
        parent = parent.GetParent()
    return parent


def update(lop_network: hou.LopNetwork):
    """Update debug isolate selected to current state in lop network."""""
    lop_node = lop_network.displayNode()
    selection: Tuple[str] = lop_network.selection()
    if selection:
        with lop_network.editablePostLayer(POST_LAYER_NAME, lop_node) as pl:

            layer = pl.layer()
            layer.ImportFromString(BLACK_SHADER)

            stage = pl.stage()
            black_material = stage.GetPrimAtPath("/__debug_black_mat")
            black_material = UsdShade.Material(black_material)

            selection_lookup = {Sdf.Path(path) for path in selection}
            prims = [stage.GetPrimAtPath(path) for path in selection]

            shader_prim = next(
                (prim for prim in prims if prim.IsA(UsdShade.Shader)),
                None
            )
            if shader_prim:
                # If Shader, find parent material and connect the shader
                # output to the material's `outputs:surface` slot.
                # If the material also has e.g. `outputs:arnold:surface`
                # attribute, then also connect it there.
                # TODO: THis does not work for MaterialX shaders since those
                #   appear to be required to go through a 'material'
                shader_output = get_main_output(UsdShade.Shader(shader_prim))
                material = get_parent_material(shader_prim)

                for attr in [
                    "outputs:mtlx:surface",
                    "outputs:arnold:surface",
                    "outputs:surface"
                ]:
                    surface_attr = material.GetAttribute(attr)
                    if not surface_attr.IsValid():
                        continue
                    surface_output = UsdShade.Output(surface_attr)
                    surface_output.ConnectToSource(shader_output)

            # Detect if any boundable in or under selection
            sel_paths = " ".join(selection)
            rule = hou.LopSelectionRule(
                f"({sel_paths} >>) & %type:Boundable"
            )
            if rule.firstPath(stage=stage):
                # Find all other boundables and disable their materials
                # completely so that only this geometry remains as 'shaded'
                rule = hou.LopSelectionRule(
                    f"%type:Boundable - ({sel_paths} >>)"
                )
                for override_prim_path in rule.expandedPaths(stage=stage):
                    boundable_prim = stage.GetPrimAtPath(override_prim_path)
                    UsdShade.MaterialBindingAPI(boundable_prim).Bind(
                        black_material)
    else:
        lop_network.removePostLayer(POST_LAYER_NAME)


def _solaris_debug_mode_on_ui_selection_changed(selection):
    for node in selection:
        if node.type().category() != hou.vopNodeTypeCategory():
            continue

        creator = node.creator()
        if creator.type().name() != "materiallibrary":
            continue

        modified = creator.lastModifiedPrims()
        if not modified:
            # No materials generated
            continue

        # The VOP node is part of a materiallibrary
        # We should now try and get its USD Prim path
        # for the selected shader or material to see
        # if it is inside the USD data. If so, we will
        # select it in the scene graph to trigger the
        # 'isolate select' callback
        node_path = node.path()
        shader_path = None
        for i in range(creator.evalParm("materials")):
            index = i + 1
            mat_node = creator.parm(f"matnode{index}").evalAsNode()
            if not mat_node:
                continue

            if not node.path().startswith(mat_node.path()):
                # It's not the material node or a child of it
                continue

            relative_path = mat_node.relativePathTo(node)
            material_path = creator.evalParm(f"matpath{index}")
            material_path_prefix = creator.evalParm(f"matpathprefix")
            if material_path.startswith("/"):
                # Absolute path set for the material
                material_path_prefix = ""

            usd_path = f"{material_path_prefix}{material_path}"
            if relative_path != ".":
                usd_path = f"{usd_path}/{relative_path}"

            # TODO: Custom HDA or some subnet will not convert with
            #   their own output in the USD data, but instead just
            #   become a parent prim and the subnet's output
            #   just get passed through - so we might need to find
            #   the child networks' output instead

            shader_prim = creator.stage().GetPrimAtPath(usd_path)
            if not shader_prim or not shader_prim.IsValid():
                continue

            shader_path = usd_path
            break

        if not shader_path:
            continue

        # TODO: remove existing Shaders from Selection
        #   to only replace the Shader selection
        network = creator.network()
        # selection = list(network.selection())
        network.setSelection([shader_path])


def _solaris_debug_mode_on_selection_changed(node, event_type):
    """Callback on """
    update(node)


def setup_callbacks(lop_network: hou.LopNetwork):
    """Setup the selection changed callbacks"""
    lop_network.addEventCallback(
        (hou.nodeEventType.SelectionChanged,),
        _solaris_debug_mode_on_selection_changed
    )
    hou.ui.addSelectionCallback(_solaris_debug_mode_on_ui_selection_changed)


def remove_callbacks(lop_network: hou.LopNetwork):
    """Remove the selection changed callbacks

    Because we're using this as a shelf script without external python file
    we don't really have a global reference to the original callbacks. So we
    just find them by name and remove that way instead.

    """
    event_types = (hou.nodeEventType.SelectionChanged,)
    callback_name = _solaris_debug_mode_on_selection_changed.__name__
    ui_callback_name = _solaris_debug_mode_on_ui_selection_changed.__name__
    for callback_event_types, callback in lop_network.eventCallbacks():

        if event_types != callback_event_types:
            continue

        if callback.__name__ == callback_name:
            lop_network.removeEventCallback(event_types, callback)

    for callback in hou.ui.selectionCallbacks():
        if callback.__name__ == ui_callback_name:
            hou.ui.removeSelectionCallback(callback)


def main():
    lop_network: hou.LopNetwork = hou.node("/stage")
    
    scene_viewer = toolutils.sceneViewer()

    # Disable with Control+Click
    remove_callbacks(lop_network)
    if kwargs.get("ctrlclick"):  # noqa: `kwargs` is defined for shelf scripts
        lop_network.removePostLayer(POST_LAYER_NAME)
        scene_viewer.setPromptMessage("Disabled Isolate Selected debug shading")
        return
    setup_callbacks(lop_network)
    scene_viewer.setPromptMessage(
        "Entered Isolate Selected debug shading mode (Ctrl+Click Shelf button to disable)", 
    )
    update(lop_network)


main()

How to use

1. Use the Python script as Houdini shelf tool script
2. Click the shelf button to enable the 'isolate selected debug shader' mode.
3. CTRL + Click the shelf button to exit the isolation mode.

Further work

It could be interesting potentially to turn this into a Viewport Python State so that you can have visual cues of the active state, and maybe even specialized selection features to isolate shaders in different ways or even pick what of the material's inputs to isolate. Lots of ideas.

It's also good to mention that the implementation here is tested mostly with Arnold Material Builder and likely won't work for some others due to how it connects the output of a given shader directly to the material which e.g. doesn't work for Materialx Builder. That's likely fixable, but not implemented.

Note: It currently uses Post Lop Layers which do get written out to an output file from USD render rops, it might be an improvement to instead apply it in the ViewportOverride session layer.