Getting NV12 plane data out of a WinRT IDirect3DSurface

Copilot Prompts.md

Get data

I have a Windows.Graphics.DirectX.Direct3D11.IDirect3DSurface - how do I get a pointer to the graphics data to operate on? Write a C++/winrt sample if you can.

Get NV12

OK, next up, split the locked bytes into a std::vector<> of NV12 plane definitions.

nv12fromd3dsurface.cpp

#include <windows.h>
#include <winrt/Windows.Graphics.DirectX.Direct3D11.h>
#include <Windows.Graphics.DirectX.Direct3D11.interop.h>
#include <d3d11.h>
#include <wil/com.h>
#include <wil/result.h>

using namespace winrt;
using namespace Windows::Graphics::DirectX::Direct3D11;

// Function to get CPU-accessible pointer from IDirect3DSurface
HRESULT GetSurfaceDataPointer(
    const IDirect3DSurface& surface,
    _Outptr_ void** ppData,
    _Out_ UINT* pRowPitch,
    _Out_ UINT* pDepthPitch)
{
    *ppData = nullptr;
    *pRowPitch = 0;
    *pDepthPitch = 0;

    // Get the native ID3D11Texture2D from the WinRT surface
    wil::com_ptr<::Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess> dxgiInterfaceAccess;
    RETURN_IF_FAILED(surface.as(dxgiInterfaceAccess));

    wil::com_ptr<ID3D11Texture2D> d3dTexture;
    RETURN_IF_FAILED(dxgiInterfaceAccess->GetInterface(IID_PPV_ARGS(&d3dTexture)));

    // Get the D3D11 device context
    wil::com_ptr<ID3D11Device> d3dDevice;
    d3dTexture->GetDevice(&d3dDevice);

    wil::com_ptr<ID3D11DeviceContext> d3dContext;
    d3dDevice->GetImmediateContext(&d3dContext);

    // Get texture description to check if it's CPU accessible
    D3D11_TEXTURE2D_DESC desc;
    d3dTexture->GetDesc(&desc);

    wil::com_ptr<ID3D11Texture2D> stagingTexture;

    // If the texture is not CPU accessible, create a staging texture
    if ((desc.Usage != D3D11_USAGE_STAGING) || 
        !(desc.CPUAccessFlags & (D3D11_CPU_ACCESS_READ | D3D11_CPU_ACCESS_WRITE)))
    {
        // Create staging texture with CPU read access
        D3D11_TEXTURE2D_DESC stagingDesc = desc;
        stagingDesc.Usage = D3D11_USAGE_STAGING;
        stagingDesc.CPUAccessFlags = D3D11_CPU_ACCESS_READ | D3D11_CPU_ACCESS_WRITE;
        stagingDesc.BindFlags = 0;
        stagingDesc.MiscFlags = 0;

        RETURN_IF_FAILED(d3dDevice->CreateTexture2D(&stagingDesc, nullptr, &stagingTexture));

        // Copy the original texture to staging texture
        d3dContext->CopyResource(stagingTexture.get(), d3dTexture.get());
    }
    else
    {
        stagingTexture = d3dTexture;
    }

    // Map the texture to get CPU access
    D3D11_MAPPED_SUBRESOURCE mappedResource;
    RETURN_IF_FAILED(d3dContext->Map(
        stagingTexture.get(),
        0, // subresource
        D3D11_MAP_READ_WRITE,
        0, // map flags
        &mappedResource));

    *ppData = mappedResource.pData;
    *pRowPitch = mappedResource.RowPitch;
    *pDepthPitch = mappedResource.DepthPitch;

    // Note: You must call Unmap when done with the data
    // d3dContext->Unmap(stagingTexture.get(), 0);

    return S_OK;
}

// RAII wrapper for automatic unmapping
class SurfaceMapper
{
public:
    SurfaceMapper() = default;
    
    ~SurfaceMapper()
    {
        if (m_context && m_texture)
        {
            m_context->Unmap(m_texture.get(), 0);
        }
    }

    HRESULT MapSurface(
        const IDirect3DSurface& surface,
        _Outptr_ void** ppData,
        _Out_ UINT* pRowPitch,
        _Out_ UINT* pDepthPitch)
    {
        *ppData = nullptr;
        *pRowPitch = 0;
        *pDepthPitch = 0;

        // Get the native ID3D11Texture2D from the WinRT surface
        wil::com_ptr<::Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess> dxgiInterfaceAccess;
        RETURN_IF_FAILED(surface.as(dxgiInterfaceAccess));

        wil::com_ptr<ID3D11Texture2D> d3dTexture;
        RETURN_IF_FAILED(dxgiInterfaceAccess->GetInterface(IID_PPV_ARGS(&d3dTexture)));

        // Get the D3D11 device context
        wil::com_ptr<ID3D11Device> d3dDevice;
        d3dTexture->GetDevice(&d3dDevice);
        d3dDevice->GetImmediateContext(&m_context);

        // Get texture description
        D3D11_TEXTURE2D_DESC desc;
        d3dTexture->GetDesc(&desc);

        // Check if we need a staging texture
        if ((desc.Usage != D3D11_USAGE_STAGING) || 
            !(desc.CPUAccessFlags & (D3D11_CPU_ACCESS_READ | D3D11_CPU_ACCESS_WRITE)))
        {
            // Create staging texture
            D3D11_TEXTURE2D_DESC stagingDesc = desc;
            stagingDesc.Usage = D3D11_USAGE_STAGING;
            stagingDesc.CPUAccessFlags = D3D11_CPU_ACCESS_READ | D3D11_CPU_ACCESS_WRITE;
            stagingDesc.BindFlags = 0;
            stagingDesc.MiscFlags = 0;

            RETURN_IF_FAILED(d3dDevice->CreateTexture2D(&stagingDesc, nullptr, &m_texture));
            m_context->CopyResource(m_texture.get(), d3dTexture.get());
        }
        else
        {
            m_texture = d3dTexture;
        }

        // Map the texture
        D3D11_MAPPED_SUBRESOURCE mappedResource;
        RETURN_IF_FAILED(m_context->Map(
            m_texture.get(),
            0,
            D3D11_MAP_READ_WRITE,
            0,
            &mappedResource));

        *ppData = mappedResource.pData;
        *pRowPitch = mappedResource.RowPitch;
        *pDepthPitch = mappedResource.DepthPitch;

        return S_OK;
    }

private:
    wil::com_ptr<ID3D11DeviceContext> m_context;
    wil::com_ptr<ID3D11Texture2D> m_texture;
};

// Example usage function
void ProcessSurfaceData(const IDirect3DSurface& surface)
{
    try
    {
        SurfaceMapper mapper;
        void* pData = nullptr;
        UINT rowPitch = 0;
        UINT depthPitch = 0;

        THROW_IF_FAILED(mapper.MapSurface(surface, &pData, &rowPitch, &depthPitch));

        // Now you can access the pixel data
        // The layout depends on the surface format (BGRA8, RGB32F, etc.)
        auto pixelData = static_cast<uint8_t*>(pData);
        
        // Example: Process pixels row by row
        // Note: You need to know the surface dimensions and format
        for (UINT row = 0; row < /* height */; ++row)
        {
            auto rowData = pixelData + (row * rowPitch);
            // Process pixels in this row
            // Format-specific processing goes here
        }

        // The mapper automatically unmaps when it goes out of scope
    }
    catch (...)
    {
        // Handle errors
        LOG_CAUGHT_EXCEPTION();
    }
}

// Alternative approach using gsl::span for safer buffer access
#include <gsl/span>
#include <vector>

template<typename PixelType>
gsl::span<PixelType> GetSurfacePixels(
    const IDirect3DSurface& surface,
    SurfaceMapper& mapper,
    UINT width,
    UINT height)
{
    void* pData = nullptr;
    UINT rowPitch = 0;
    UINT depthPitch = 0;

    THROW_IF_FAILED(mapper.MapSurface(surface, &pData, &rowPitch, &depthPitch));

    // Calculate total pixel count
    // Note: This assumes tightly packed pixels, which may not always be true due to rowPitch
    size_t pixelCount = static_cast<size_t>(width) * height;
    
    return gsl::make_span(static_cast<PixelType*>(pData), pixelCount);
}

// NV12 plane definition
struct NV12Plane
{
    gsl::span<uint8_t> data;
    UINT width;
    UINT height;
    UINT rowPitch;
    UINT pixelStride;  // 1 for Y plane, 2 for UV plane
};

// NV12 format structure
struct NV12Surface
{
    NV12Plane yPlane;   // Luminance plane (Y)
    NV12Plane uvPlane;  // Chrominance plane (interleaved U and V)
    UINT surfaceWidth;
    UINT surfaceHeight;
};

// Split NV12 surface data into plane definitions
NV12Surface SplitNV12Surface(
    gsl::span<uint8_t> surfaceData,
    UINT width,
    UINT height,
    UINT rowPitch)
{
    // NV12 format layout:
    // - Y plane: full resolution (width x height), 1 byte per pixel
    // - UV plane: half resolution (width/2 x height/2), 2 bytes per pixel (interleaved U,V)
    
    THROW_HR_IF(E_INVALIDARG, width == 0 || height == 0);
    THROW_HR_IF(E_INVALIDARG, width % 2 != 0 || height % 2 != 0); // NV12 requires even dimensions
    
    NV12Surface result{};
    result.surfaceWidth = width;
    result.surfaceHeight = height;
    
    // Y plane (luminance) - starts at beginning of buffer
    UINT yPlaneSize = rowPitch * height;
    result.yPlane.data = surfaceData.subspan(0, yPlaneSize);
    result.yPlane.width = width;
    result.yPlane.height = height;
    result.yPlane.rowPitch = rowPitch;
    result.yPlane.pixelStride = 1;
    
    // UV plane (chrominance) - starts after Y plane
    // UV plane has half the height and width, but 2 bytes per "pixel" (U and V interleaved)
    UINT uvPlaneHeight = height / 2;
    UINT uvPlaneWidth = width / 2;
    UINT uvRowPitch = rowPitch; // Usually same as Y plane, but contains interleaved U,V
    UINT uvPlaneSize = uvRowPitch * uvPlaneHeight;
    
    THROW_HR_IF(E_INVALIDARG, yPlaneSize + uvPlaneSize > surfaceData.size());
    
    result.uvPlane.data = surfaceData.subspan(yPlaneSize, uvPlaneSize);
    result.uvPlane.width = uvPlaneWidth;
    result.uvPlane.height = uvPlaneHeight;
    result.uvPlane.rowPitch = uvRowPitch;
    result.uvPlane.pixelStride = 2; // U and V are interleaved
    
    return result;
}

// Convenience function to get NV12 planes from a Direct3D surface
NV12Surface GetNV12SurfaceFromDirect3D(const IDirect3DSurface& surface)
{
    // First, get the surface description to validate it's NV12
    wil::com_ptr<::Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess> dxgiInterfaceAccess;
    THROW_IF_FAILED(surface.as(dxgiInterfaceAccess));

    wil::com_ptr<ID3D11Texture2D> d3dTexture;
    THROW_IF_FAILED(dxgiInterfaceAccess->GetInterface(IID_PPV_ARGS(&d3dTexture)));

    D3D11_TEXTURE2D_DESC desc;
    d3dTexture->GetDesc(&desc);
    
    // Validate this is an NV12 surface
    THROW_HR_IF(E_INVALIDARG, desc.Format != DXGI_FORMAT_NV12);
    
    // Map the surface to get CPU access
    SurfaceMapper mapper;
    void* pData = nullptr;
    UINT rowPitch = 0;
    UINT depthPitch = 0;
    
    THROW_IF_FAILED(mapper.MapSurface(surface, &pData, &rowPitch, &depthPitch));
    
    // Calculate total surface size for NV12
    // Y plane: rowPitch * height
    // UV plane: rowPitch * (height/2)
    UINT totalSize = rowPitch * desc.Height + rowPitch * (desc.Height / 2);
    
    auto surfaceSpan = gsl::make_span(static_cast<uint8_t*>(pData), totalSize);
    
    return SplitNV12Surface(surfaceSpan, desc.Width, desc.Height, rowPitch);
}

// Helper functions to access individual color components
inline uint8_t GetYValue(const NV12Plane& yPlane, UINT x, UINT y)
{
    THROW_HR_IF(E_INVALIDARG, x >= yPlane.width || y >= yPlane.height);
    return yPlane.data[y * yPlane.rowPitch + x];
}

inline std::pair<uint8_t, uint8_t> GetUVValues(const NV12Plane& uvPlane, UINT x, UINT y)
{
    THROW_HR_IF(E_INVALIDARG, x >= uvPlane.width || y >= uvPlane.height);
    UINT offset = y * uvPlane.rowPitch + x * 2;
    uint8_t u = uvPlane.data[offset];
    uint8_t v = uvPlane.data[offset + 1];
    return {u, v};
}

// Example processing function
void ProcessNV12Surface(const IDirect3DSurface& surface)
{
    try
    {
        auto nv12 = GetNV12SurfaceFromDirect3D(surface);
        
        // Process Y plane (luminance)
        for (UINT y = 0; y < nv12.yPlane.height; ++y)
        {
            auto rowStart = nv12.yPlane.data.data() + (y * nv12.yPlane.rowPitch);
            auto rowSpan = gsl::make_span(rowStart, nv12.yPlane.width);
            
            // Process luminance values in this row
            for (uint8_t& yValue : rowSpan)
            {
                // Example: Apply brightness adjustment
                yValue = static_cast<uint8_t>(std::min(255, static_cast<int>(yValue) + 10));
            }
        }
        
        // Process UV plane (chrominance)
        for (UINT y = 0; y < nv12.uvPlane.height; ++y)
        {
            auto rowStart = nv12.uvPlane.data.data() + (y * nv12.uvPlane.rowPitch);
            
            for (UINT x = 0; x < nv12.uvPlane.width; ++x)
            {
                UINT offset = x * 2;
                uint8_t& u = rowStart[offset];
                uint8_t& v = rowStart[offset + 1];
                
                // Example: Apply color adjustment
                // Reduce saturation slightly
                u = static_cast<uint8_t>(128 + (static_cast<int>(u) - 128) * 0.9);
                v = static_cast<uint8_t>(128 + (static_cast<int>(v) - 128) * 0.9);
            }
        }
    }
    catch (...)
    {
        LOG_CAUGHT_EXCEPTION();
    }
}