IOSCamera.cpp

#include "IOSCamera.h"
#include "IMediaTextureSink.h"
#include "MediaTexture.h"


@interface FSampleBufferDelegate: NSObject<AVCaptureVideoDataOutputSampleBufferDelegate>
@property (assign) FIOSCamera* Target;
@end

@implementation FSampleBufferDelegate

- (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection
{
    _Target->HandleSampleBuffer(sampleBuffer);
}

@end

namespace {
  /**
   * Passes a CV*TextureRef or CVPixelBufferRef through to the RHI to wrap in an RHI texture without traversing system memory.
   */
  class FAvfTexture2DResourceWrapper : public FResourceBulkDataInterface
  {
  public:
    FAvfTexture2DResourceWrapper(CFTypeRef InImageBuffer)
      : ImageBuffer(InImageBuffer)
    {
      check(ImageBuffer);
      CFRetain(ImageBuffer);
    }
    
    /**
     * @return ptr to the resource memory which has been preallocated
     */
    virtual const void* GetResourceBulkData() const override
    {
      return ImageBuffer;
    }
    
    /**
     * @return size of resource memory
     */
    virtual uint32 GetResourceBulkDataSize() const override
    {
      return ImageBuffer ? ~0u : 0;
    }
    
    /**
     * Free memory after it has been used to initialize RHI resource
     */
    virtual void Discard() override
    {
      delete this;
    }
    virtual ~FAvfTexture2DResourceWrapper()
    {
      CFRelease(ImageBuffer);
      ImageBuffer = nullptr;
    }
    
    CFTypeRef ImageBuffer;
  };
}

FIOSCamera::FIOSCamera(UMediaTexture* InSink): VideoSink(InSink)
{
    VideoSink->AddToRoot();
    SampleBufferDelegate = [[FSampleBufferDelegate alloc] init];
    SampleBufferDelegate.Target = this;
    TextureCache = nil;
    AVSession = nil;
    VideoDevice = nil;
    DispatchQueue = nil;
}

FIOSCamera::~FIOSCamera()
{
    [AVSession release];
    AVSession = nil;
    TextureCache = nil;
    VideoDevice = nil;
    [DispatchQueue release];
    DispatchQueue = nil;
    [SampleBufferDelegate release];
    SampleBufferDelegate = nil;
    VideoSink->RemoveFromRoot();
}

void FIOSCamera::SetupCamera()
{
    FIntPoint Dimensions(1280, 720);
    
    VideoSink->InitializeTextureSink(Dimensions,
                                     Dimensions,
                                     EMediaTextureSinkFormat::CharBGRA,
                                     EMediaTextureSinkMode::Unbuffered);
    VideoSink->UpdateResource();
    if (AVSession == nil)
    {
        AVSession = [[AVCaptureSession alloc] init];
        [AVSession beginConfiguration];
        if ([AVSession canSetSessionPreset:AVCaptureSessionPreset1280x720])
        {
            [AVSession setSessionPreset:AVCaptureSessionPreset1280x720];
        }
        else
        {
            UE_LOG(LogTemp, Error, TEXT("Couldn't set session preset 1280x720"));
        }
        VideoDevice = [AVCaptureDevice defaultDeviceWithDeviceType:AVCaptureDeviceTypeBuiltInDualCamera mediaType:AVMediaTypeVideo position:AVCaptureDevicePositionBack];
        if (VideoDevice == nil)
        {
            VideoDevice = [AVCaptureDevice defaultDeviceWithDeviceType:AVCaptureDeviceTypeBuiltInWideAngleCamera mediaType:AVMediaTypeVideo position:AVCaptureDevicePositionBack];
        }
        NSError * error = nil;
        AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:VideoDevice error:&error];
        if (input == nil)
        {
            NSLog(@"%s - %@", __PRETTY_FUNCTION__, error);
        }
        [AVSession addInput:input];
        DispatchQueue = dispatch_queue_create("CameraMulticaster", DISPATCH_QUEUE_SERIAL);
        AVCaptureVideoDataOutput* dataOutput = [[AVCaptureVideoDataOutput alloc] init];
        [dataOutput setAlwaysDiscardsLateVideoFrames:YES];
        [dataOutput setVideoSettings:@{(id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA)}];
        [dataOutput setSampleBufferDelegate:SampleBufferDelegate queue:DispatchQueue];
        [AVSession addOutput:dataOutput];
        [AVSession commitConfiguration];
        UE_LOG(LogTemp, Log, TEXT("Setup IOS Camera"));
        
    }
    [AVSession startRunning];
}

void FIOSCamera::TeardownCamera()
{
    VideoSink->ShutdownTextureSink();
    [AVSession stopRunning];
}

void FIOSCamera::HandleSampleBuffer(CMSampleBufferRef sampleBuffer)
{
    CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
    size_t Width = CVPixelBufferGetWidth(pixelBuffer);
    size_t Height = CVPixelBufferGetHeight(pixelBuffer);
    if (TextureCache == nil)
    {
        id<MTLDevice> MetalDevice = (id<MTLDevice>)GDynamicRHI->RHIGetNativeDevice();
        CVMetalTextureCacheCreate(NULL, NULL, MetalDevice, NULL, &TextureCache);
    }
    CVMetalTextureRef texture = NULL;
    CVReturn status = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, TextureCache, pixelBuffer, NULL, MTLPixelFormatBGRA8Unorm_sRGB, Width, Height, 0, &texture);
    if(status == kCVReturnSuccess)
    {
        auto Wrapper = new FAvfTexture2DResourceWrapper(texture);
        CFRelease(texture);
        CMTime timestamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);
        FTimespan DisplayTime = FTimespan::FromSeconds(CMTimeGetSeconds(timestamp));
        ENQUEUE_UNIQUE_RENDER_COMMAND_FIVEPARAMETER(UpdateSinkCommand,
                                                     FAvfTexture2DResourceWrapper*, Wrapper, Wrapper,
                                                     UMediaTexture*, VideoSink, VideoSink,
                                                     FTimespan, DisplayTime, DisplayTime,
                                                     size_t, Width, Width,
                                                     size_t, Height, Height,
                                                     {
                                                         FRHIResourceCreateInfo CreateInfo;
                                                         CreateInfo.BulkData = Wrapper;
                                                         CreateInfo.ResourceArray = nullptr;
                                                         uint32 TexCreateFlags = TexCreate_SRGB;
                                                         TexCreateFlags |= TexCreate_Dynamic | TexCreate_NoTiling;
                                                         TRefCountPtr<FRHITexture2D> ShaderResource = RHICreateTexture2D(Width, Height, PF_B8G8R8A8, 1, 1, TexCreateFlags | TexCreate_ShaderResource, CreateInfo);
                                                         VideoSink->UpdateTextureSinkResource(ShaderResource, ShaderResource);
                                                         //UE_LOG(LogTemp, Log, TEXT("UpdateTextureSinkResource ShaderResource Refc=%d"), ShaderResource->GetRefCount());
                                                     });
    }
    else
    {
        UE_LOG(LogTemp, Error, TEXT("IOSCamera:: Couldn't get texture from cache"));
    }
}

FVector2D FIOSCamera::GetVideoSize()
{
    if (VideoDevice == nil)
    {
        return FVector2D(1280, 720);
    }
    auto dim = CMVideoFormatDescriptionGetDimensions(VideoDevice.activeFormat.formatDescription);
    return FVector2D(dim.width, dim.height);
}


float FIOSCamera::GetFieldOfView()
{
    return VideoDevice != nil ? VideoDevice.activeFormat.videoFieldOfView : 90;
}