gistfile1.cs

public static Vector3 ProjectAOntoB(Vector3 A, Vector3 B)
        {
            return Vector3.Dot(A, B) * B;
        }

        public override void HandleMouse(Ray mouseRay, Action<VertexPositionColor, VertexPositionColor> debug)
        {
            //Transform triangle into world space
            var verts = new Vector3[3];
            verts[0] = new Vector3(-0.5f, -0.5f, 0);
            verts[1] = new Vector3(0.5f, -0.5f, 0);
            verts[2] = new Vector3(-0.5f, 0.5f, 0);

            for (int i = 0; i < 3; ++i)
                verts[i] = Vector3.Transform(verts[i], localTransformation);

            //Find intersection point with plane
            var distance = mouseRay.Intersects(new Plane(verts[0], verts[1], verts[2]));
            if (distance == null || !distance.HasValue) return;
            var interesectionPoint = mouseRay.Position + (mouseRay.Direction * distance.Value);

            //Here's where the signs are lost - Length();
            var x = ProjectAOntoB(interesectionPoint - verts[0], verts[1] - verts[0]).Length();
            var y = ProjectAOntoB(interesectionPoint - verts[0], verts[2] - verts[0]).Length();
            
            //Convert to local space and handle
            uiRoot.HandleMouse((int)(x * uiCamera.viewportDimensions.X), (int)(y * uiCamera.viewportDimensions.Y));
        }