Check for any pre-existing hooks.

Library.cpp

#define INITGUID
#define COBJMACROS
#define WIDL_C_INLINE_WRAPPERS

#include <windows.h>
#include <MinHook.h>
#include <d3d11.h>

#include <winrt/base.h>
#include <winrt/Windows.UI.Core.h>
#include <winrt/Windows.UI.Popups.h>
#include <winrt/Windows.Foundation.h>
#include <winrt/Windows.ApplicationModel.Core.h>

using namespace winrt;
using namespace Windows::UI::Core;
using namespace Windows::Foundation;
using namespace Windows::UI::Popups;
using namespace Windows::ApplicationModel::Core;

const std::unordered_map<std::wstring, std::wstring> _ = {{L"Rivatuner Statistics Server", L"RTSSHooks64.dll"},
                                                          {L"OBS Studio", L"graphics-hook64.dll"}};
VOID MessageDialogW(PCWSTR pText, PCWSTR pTitle)
{
    IAsyncOperation<IUICommand> _ = nullptr;
    if (CoreApplication::MainView()
            .CoreWindow()
            .Dispatcher()
            .TryRunAsync(CoreDispatcherPriority::High,
                         [pText, pTitle, &_]() {
                             _ = (pTitle ? MessageDialog(pText, pTitle) : MessageDialog(pText)).ShowAsync();
                         })
            .get())
        _.get();
}


// Check if the passed function pointer for any form of hooking.
bool IsHooked(PVOID pFunction)
{
    auto pAddress = reinterpret_cast<PBYTE>(pFunction);
    return pAddress[0] == 0xE9 || (pAddress[0] == 0x68 && pAddress[5] == 0xC3);
}

// Check if the passed function pointer is hooked & check for specific modules that might have hooked the passed function.
bool IsHookedByModule(PVOID pFunction)
{
    if (!IsHooked(pFunction))
        return false;

    for (std::pair<std::wstring, std::wstring> _ : _)
        if (GetModuleHandleW(_.second.c_str()))
        {
            MessageDialogW(std::format(L"Library cannot initialize due to {}.", _.first).c_str(), nullptr);
            return true;
        }

    return false;
}

DWORD ThreadProc(PVOID pParameter)
{
    DXGI_SWAP_CHAIN_DESC Desc{};
    Desc.BufferCount = 1;
    Desc.BufferDesc.Format = DXGI_FORMAT_B8G8R8A8_UNORM;
    Desc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
    Desc.SampleDesc.Count = 1;
    Desc.Windowed = TRUE;
    Desc.OutputWindow = GetDesktopWindow();

    IDXGISwapChain *pSwapChain{};

    D3D11CreateDeviceAndSwapChain(NULL, D3D_DRIVER_TYPE_HARDWARE, NULL, (UINT){}, NULL, (UINT){}, D3D11_SDK_VERSION,
                                  &Desc, &pSwapChain, NULL, NULL, NULL);

    IsHookedByModule((*reinterpret_cast<PVOID **>(pSwapChain))[8]);

    pSwapChain->Release();
    FreeLibraryAndExitThread(reinterpret_cast<HINSTANCE>(pParameter), EXIT_SUCCESS);
    return EXIT_SUCCESS;
}

BOOL DllMain(HINSTANCE hInstance, DWORD dwReason, PVOID pReserved)
{
    if (dwReason == DLL_PROCESS_ATTACH)
    {
        DisableThreadLibraryCalls(hInstance);
        CloseHandle(CreateThread(NULL, SIZE_T(), ThreadProc, hInstance, (DWORD){}, NULL));
    }
    return TRUE;
}