Example of getting viewport under mouse, and from there hit position and which component was closest to the hit

component_under_mouse.py

"""

    Command to print the component under the mouse using lx api in Foundry Modo

    To try it out, map the command to a hotkey and execute while mouse is over corresponding component.

        py.mouse.component type:{vert|edge|poly}

    You will likely want to use the already existing method

        lx.eval("query view3dservice element.over ? poly")

"""

from __future__ import annotations
from typing import Tuple  # to help with type hints,

from math import isclose

import lx
import lxu
import lxu.vector  # these two are not needed, but only to help my IDE find modules
import lxu.command

from modo.mathutils import Vector3


def distance_to_line(point: Tuple[float, float, float], a: Tuple[float, float, float], b: Tuple[float, float, float]) -> float:
    """ Get the distance between a point and an edge
    https://en.wikipedia.org/wiki/Distance_from_a_point_to_a_line#Vector_formulation """
    a = Vector3(*a)
    n = Vector3(*b) - a
    p = Vector3(*point)
    return ((p - a) - ((p - a) * n) * n).length()


class Command(lxu.command.BasicCommand):
    def __init__(self):
        super(Command, self).__init__()
        self.dyna_Add("type", lx.symbol.sTYPE_INTEGER)
        self.dyna_SetHint(0, ((lx.symbol.iSEL_VERTEX, "vert"), (lx.symbol.iSEL_EDGE, "edge"), (lx.symbol.iSEL_POLYGON, "poly")))

        self.dyna_Add("tolerance", lx.symbol.sTYPE_FLOAT)
        self.dyna_SetFlags(1, lx.symbol.fCMDARG_OPTIONAL)

    def cmd_DialogInit(self):
        if not self.dyna_IsSet(0):
            self.attr_SetInt(0, lx.symbol.iSEL_POLYGON)

        if not self.dyna_IsSet(1):
            self.attr_SetFlt(1, 0.01)

    def basic_Execute(self, msg, flags):
        tolerance = self.dyna_Float(1, 0.01)

        view_service = lx.service.View3Dport()
        view_index, x, y = view_service.Mouse()  # get the 3d view, and mouse position in that view
        if view_index < 0:
            return  # there is no 3d view under the mouse

        view = lx.object.View3D(view_service.View(view_index))

        # get hit 3d position for mouse, raises lookup error if not hitting anything.
        try:
            hit, _ = view.To3DHit(x, y)
        except LookupError:
            return

        value_service = lx.service.Value()

        # go through each active layer and check for closest polygon to our hit location
        layer_service = lx.service.Layer()
        layer_scan = layer_service.ScanAllocate(lx.symbol.f_LAYERSCAN_ALL)
        for layer_index in range(layer_scan.Count()):
            mesh = layer_scan.MeshBase(layer_index)
            poly = lx.object.Polygon(mesh.PolygonAccessor())
            point = lx.object.Point(mesh.PointAccessor())

            # get the local space coordinates for the hit
            xfrm = layer_scan.MeshTransform(layer_index)
            matrix = lx.object.Matrix(value_service.CreateValue(lx.symbol.sTYPE_MATRIX4))
            matrix.Set4(xfrm)
            matrix.Invert()
            local_hit = matrix.MultiplyVector(hit)

            # check if any polygons are close to the hit, Modo will select the polygon if we did hit
            did_hit, hit_position, hit_normal, hit_distance = poly.Closest(0.001, local_hit)

            # If we did hit this layer polygon, branch on component and print it's index
            if did_hit:
                if self.dyna_Int(0) == lx.symbol.iSEL_POLYGON:
                    print(f"Hit polygon: {poly.Index()}")

                elif self.dyna_Int(0) == lx.symbol.iSEL_EDGE:
                    # get all point indices for the hit polygon
                    points = []
                    for poly_vert_index in range(poly.VertexCount()):
                        point.Select(poly.VertexByIndex(poly_vert_index))
                        points.append(point.Index())

                    # edges are made out of two end points
                    edges = [(a, b) for a, b in zip(points, points[1:] + [points[0]])]

                    # for each edge, check distance to hit and if within tolerance print the end point indices
                    for (a, b) in edges:
                        point.SelectByIndex(a)
                        a_pos = point.Pos()
                        point.SelectByIndex(b)
                        b_pos = point.Pos()
                        distance = distance_to_line(local_hit, a_pos, b_pos)
                        if isclose(distance, 0.0, abs_tol=tolerance):
                            print(f"Hit edge: {a}, {b}")

                elif self.dyna_Int(0) == lx.symbol.iSEL_VERTEX:
                    # for each polygon vertex check the distance to hit is within tolerance and print the index
                    for poly_vert_index in range(poly.VertexCount()):
                        point.Select(poly.VertexByIndex(poly_vert_index))
                        position = point.Pos()
                        distance = lxu.vector.length(lxu.vector.sub(local_hit, position))
                        if isclose(distance, 0.0, abs_tol=tolerance):
                            print(f"Hit vertex: {point.Index()}")

        del layer_scan  # explicitly delete the layer object, have had issues running the same command twice otherwise


lx.bless(Command, "py.mouse.component")