Skip to content

Instantly share code, notes, and snippets.

@ara4n
Created January 25, 2018 20:42
Show Gist options
  • Save ara4n/681140784fe540a3e83bc9e034808fb2 to your computer and use it in GitHub Desktop.
Save ara4n/681140784fe540a3e83bc9e034808fb2 to your computer and use it in GitHub Desktop.
wip experimentation with AVCaptureDepthData in webrtc
diff --git a/sdk/objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.h b/sdk/objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.h
index 8b73a8086..6b23e8a81 100644
--- a/sdk/objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.h
+++ b/sdk/objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.h
@@ -24,6 +24,9 @@ NS_ASSUME_NONNULL_BEGIN
@interface RTCAVFoundationVideoCapturerInternal
: NSObject <AVCaptureVideoDataOutputSampleBufferDelegate>
+@interface RTCAVFoundationVideoCapturerInternal
+ : NSObject <AVCaptureDepthDataOutputDelegate>
+
@property(nonatomic, readonly) AVCaptureSession *captureSession;
@property(nonatomic, readonly) dispatch_queue_t frameQueue;
@property(nonatomic, readonly) BOOL canUseBackCamera;
diff --git a/sdk/objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.mm b/sdk/objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.mm
index 287ecdd71..b9cc14814 100644
--- a/sdk/objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.mm
+++ b/sdk/objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.mm
@@ -263,6 +263,61 @@
RTCLogError(@"Dropped sample buffer.");
}
+#pragma mark AVCaptureDepthDataOutputDelegate
+
+- (void)depthDataOutput:(AVCaptureDepthDataOutput *)depthDataOutput
+ didOutput:(AVDepthData *)depthData
+ timestamp:(CMTime)timestamp
+ connection:(AVCaptureConnection *)connection {
+
+ // FIXME: should we track our AVCaptureDepthDataOutput like we track our
+ // AVCaptureDepthDataOutput
+ // NSParameterAssert(depthDataOutput == _videoDataOutput);
+
+ if (!self.hasStarted) {
+ return;
+ }
+
+ // Assume that rotation metadata, if any, will be tracked by non-depth capture
+ // where we can use AVCaptureSession devicePositionForSampleBuffer correctly
+ webrtc::VideoRotation rotation = webrtc::kVideoRotation_0;
+
+ OSStatus status;
+ CMFormatDescriptionRef desc = NULL;
+ status = CMVideoFormatDescriptionCreateForImageBuffer(NULL, depthData->depthDataMap, &desc);
+ if (status != noErr) {
+ RTC_LOG(LS_ERROR) << "CMVideoFormatDescriptionCreateForImageBuffer failed to set: " << status;
+ }
+
+ CMSampleTimingInfo timing;
+ timing.duration = kCMTimeInvalid;
+ timing.presentationTimeStamp = timestamp;
+ timing.decodeTimeStamp = kCMTimeInvalid;
+
+ // Convert our depthData into a SampleBuffer
+ CMSampleBufferRef sampleBuffer;
+ status = CMSampleBufferCreateReadyWithImageBuffer(
+ kCFAllocatorDefault,
+ depthData->depthDataMap,
+ desc,
+ &timing,
+ &sampleBuffer
+ );
+ if (status != noErr) {
+ RTC_LOG(LS_ERROR) << "CMSampleBufferCreateReadyWithImageBuffer failed to set: " << status;
+ }
+
+ _capturer->CaptureSampleBuffer(sampleBuffer, rotation);
+}
+
+- (void)depthDataOutput:(AVCaptureDepthDataOutput *)output
+ didDropDepthData:(AVDepthData *)depthData
+ timestamp:(CMTime)timestamp
+ connection:(AVCaptureConnection *)connection
+ reason:(AVCaptureOutputDataDroppedReason)reason {
+ RTCLogError(@"Dropped depthData buffer.");
+}
+
#pragma mark - AVCaptureSession notifications
- (void)handleCaptureSessionInterruption:(NSNotification *)notification {
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment