ExampleEditorWindow.cs

using UnityEditor;
using UnityEditor.Overlays;
using UnityEngine;

internal sealed class ExampleEditorWindow :
    EditorWindow,
    ISupportsOverlays
{
    [MenuItem( "Tools/ExampleEditorWindow" )]
    private static void Open()
    {
        GetWindow<ExampleEditorWindow>();
    }

    private void OnGUI()
    {
        GUILayout.Label( "ピカチュウ" );
    }
}

ExampleOverlay.cs

using UnityEditor.Overlays;
using UnityEngine.UIElements;

[Overlay
(
    editorWindowType: typeof( ExampleEditorWindow ),
    displayName: "ライチュウ",
    defaultDisplay: true
)]
internal sealed class ExampleOverlay : Overlay
{
    public override VisualElement CreatePanelContent()
    {
        var label = new Label();
        label.RegisterCallback<MouseEnterEvent>( _ => label.text = "マウスが入った" );
        label.RegisterCallback<MouseLeaveEvent>( _ => label.text = "マウスが出た" );
        return label;
    }
}