Merge pull request #6573 from fabianishere/feat/mac-virtualcam

mac-virtualcam: Avoid transcoding where possible

plugins/mac-virtualcam/src/dal-plugin/CMSampleBufferUtils.h

@@ -7,16 +7,6 @@
 
 #include <CoreMediaIO/CMIOSampleBuffer.h>
 
-OSStatus CMSampleBufferCreateFromData(NSSize size,
-				      CMSampleTimingInfo timingInfo,
-				      UInt64 sequenceNumber, NSData *data,
-				      CMSampleBufferRef *sampleBuffer);
-
-OSStatus CMSampleBufferCreateFromDataNoCopy(NSSize size,
-					    CMSampleTimingInfo timingInfo,
-					    UInt64 sequenceNumber, NSData *data,
-					    CMSampleBufferRef *sampleBuffer);
-
 CMSampleTimingInfo CMSampleTimingInfoForTimestamp(uint64_t timestampNanos,
 						  uint32_t fpsNumerator,
 						  uint32_t fpsDenominator);

plugins/mac-virtualcam/src/dal-plugin/CMSampleBufferUtils.mm

@@ -7,165 +7,6 @@
 
 #import "CMSampleBufferUtils.h"
 
-#include "Logging.h"
-
-/*!
-CMSampleBufferCreateFromData
-
-Creates a CMSampleBuffer by copying bytes from NSData into a CVPixelBuffer.
-*/
-OSStatus CMSampleBufferCreateFromData(NSSize size,
-				      CMSampleTimingInfo timingInfo,
-				      UInt64 sequenceNumber, NSData *data,
-				      CMSampleBufferRef *sampleBuffer)
-{
-	OSStatus err = noErr;
-
-	// Create an empty pixel buffer
-	CVPixelBufferRef pixelBuffer;
-	err = CVPixelBufferCreate(kCFAllocatorDefault, size.width, size.height,
-				  kCVPixelFormatType_422YpCbCr8, nil,
-				  &pixelBuffer);
-	if (err != noErr) {
-		DLog(@"CVPixelBufferCreate err %d", err);
-		return err;
-	}
-
-	// Generate the video format description from that pixel buffer
-	CMFormatDescriptionRef format;
-	err = CMVideoFormatDescriptionCreateForImageBuffer(NULL, pixelBuffer,
-							   &format);
-	if (err != noErr) {
-		DLog(@"CMVideoFormatDescriptionCreateForImageBuffer err %d",
-		     err);
-		return err;
-	}
-
-	// Copy memory into the pixel buffer
-	CVPixelBufferLockBaseAddress(pixelBuffer, 0);
-	uint8_t *dest =
-		(uint8_t *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);
-	uint8_t *src = (uint8_t *)data.bytes;
-
-	size_t destBytesPerRow =
-		CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
-	size_t srcBytesPerRow = size.width * 2;
-
-	// Sometimes CVPixelBufferCreate will create a pixelbuffer that's a different
-	// size than necessary to hold the frame (probably for some optimization reason).
-	// If that is the case this will do a row-by-row copy into the buffer.
-	if (destBytesPerRow == srcBytesPerRow) {
-		memcpy(dest, src, data.length);
-	} else {
-		for (int line = 0; line < size.height; line++) {
-			memcpy(dest, src, srcBytesPerRow);
-			src += srcBytesPerRow;
-			dest += destBytesPerRow;
-		}
-	}
-
-	CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
-
-	err = CMIOSampleBufferCreateForImageBuffer(kCFAllocatorDefault,
-						   pixelBuffer, format,
-						   &timingInfo, sequenceNumber,
-						   0, sampleBuffer);
-	CFRelease(format);
-	CFRelease(pixelBuffer);
-
-	if (err != noErr) {
-		DLog(@"CMIOSampleBufferCreateForImageBuffer err %d", err);
-		return err;
-	}
-
-	return noErr;
-}
-
-static void releaseNSData(void *o, void *, size_t)
-{
-	NSData *data = (__bridge_transfer NSData *)o;
-	data = nil; // Assuming ARC is enabled
-}
-
-// From https://stackoverflow.com/questions/26158253/how-to-create-a-cmblockbufferref-from-nsdata
-OSStatus createReadonlyBlockBuffer(CMBlockBufferRef *result, NSData *data)
-{
-	CMBlockBufferCustomBlockSource blockSource = {
-		.version = kCMBlockBufferCustomBlockSourceVersion,
-		.AllocateBlock = NULL,
-		.FreeBlock = &releaseNSData,
-		.refCon = (__bridge_retained void *)data,
-	};
-	return CMBlockBufferCreateWithMemoryBlock(NULL, (void *)data.bytes,
-						  data.length, NULL,
-						  &blockSource, 0, data.length,
-						  0, result);
-}
-
-/*!
- CMSampleBufferCreateFromDataNoCopy
-
- Creates a CMSampleBuffer by using the bytes directly from NSData (without copying them).
- Seems to mostly work but does not work at full resolution in OBS for some reason (which prevents loopback testing).
- */
-OSStatus CMSampleBufferCreateFromDataNoCopy(NSSize size,
-					    CMSampleTimingInfo timingInfo,
-					    UInt64 sequenceNumber, NSData *data,
-					    CMSampleBufferRef *sampleBuffer)
-{
-	OSStatus err = noErr;
-
-	CMBlockBufferRef dataBuffer;
-	createReadonlyBlockBuffer(&dataBuffer, data);
-
-	// Magic format properties snagged from https://github.com/lvsti/CoreMediaIO-DAL-Example/blob/0392cbf27ed33425a1a5bd9f495b2ccec8f20501/Sources/Extras/CoreMediaIO/DeviceAbstractionLayer/Devices/Sample/PlugIn/CMIO_DP_Sample_Stream.cpp#L830
-	NSDictionary *extensions = @{
-		@"com.apple.cmio.format_extension.video.only_has_i_frames":
-			@YES,
-		(__bridge NSString *)
-		kCMFormatDescriptionExtension_FieldCount: @1,
-		(__bridge NSString *)
-		kCMFormatDescriptionExtension_ColorPrimaries:
-			(__bridge NSString *)
-				kCMFormatDescriptionColorPrimaries_SMPTE_C,
-		(__bridge NSString *)
-		kCMFormatDescriptionExtension_TransferFunction: (
-			__bridge NSString *)
-			kCMFormatDescriptionTransferFunction_ITU_R_709_2,
-		(__bridge NSString *)
-		kCMFormatDescriptionExtension_YCbCrMatrix: (__bridge NSString *)
-			kCMFormatDescriptionYCbCrMatrix_ITU_R_601_4,
-		(__bridge NSString *)
-		kCMFormatDescriptionExtension_BytesPerRow: @(size.width * 2),
-		(__bridge NSString *)kCMFormatDescriptionExtension_FormatName:
-			@"Component Video - CCIR-601 uyvy",
-		(__bridge NSString *)kCMFormatDescriptionExtension_Version: @2,
-	};
-
-	CMFormatDescriptionRef format;
-	err = CMVideoFormatDescriptionCreate(
-		NULL, kCMVideoCodecType_422YpCbCr8, size.width, size.height,
-		(__bridge CFDictionaryRef)extensions, &format);
-	if (err != noErr) {
-		DLog(@"CMVideoFormatDescriptionCreate err %d", err);
-		return err;
-	}
-
-	size_t dataSize = data.length;
-	err = CMIOSampleBufferCreate(kCFAllocatorDefault, dataBuffer, format, 1,
-				     1, &timingInfo, 1, &dataSize,
-				     sequenceNumber, 0, sampleBuffer);
-	CFRelease(format);
-	CFRelease(dataBuffer);
-
-	if (err != noErr) {
-		DLog(@"CMIOSampleBufferCreate err %d", err);
-		return err;
-	}
-
-	return noErr;
-}
-
 CMSampleTimingInfo CMSampleTimingInfoForTimestamp(uint64_t timestampNanos,
 						  uint32_t fpsNumerator,
 						  uint32_t fpsDenominator)
@@ -175,7 +16,7 @@ CMSampleTimingInfo CMSampleTimingInfoForTimestamp(uint64_t timestampNanos,
 	// timestamps and scales like mach_absolute_time() and NSEC_PER_SEC will work for display, but will error out
 	// when trying to record.
 	//
-	// 600 is a commmon default in Apple's docs https://developer.apple.com/documentation/avfoundation/avmutablemovie/1390622-timescale
+	// 600 is a common default in Apple's docs https://developer.apple.com/documentation/avfoundation/avmutablemovie/1390622-timescale
 	CMTimeScale scale = 600;
 	CMSampleTimingInfo timing;
 	timing.duration =

plugins/mac-virtualcam/src/dal-plugin/CMakeLists.txt

@@ -1,15 +1,12 @@
-# Build DAL plugin universal to ensure compatibility with Rosetta-translated
-# apps on arm64 hosts
-set(CMAKE_OSX_ARCHITECTURES "x86_64;arm64")
-
 project(mac-dal-plugin)
 
 find_library(COCOA Cocoa)
 find_library(COREMEDIA CoreMedia)
 find_library(COREMEDIAIO CoreMediaIO)
 find_library(COREVIDEO CoreVideo)
+find_library(IOSURFACE IOSurface)
 
-mark_as_advanced(COCOA COREMEDIA COREMEDIAIO COREVIDEO)
+mark_as_advanced(COCOA COREMEDIA COREMEDIAIO COREVIDEO IOSURFACE)
 
 add_library(mac-dal-plugin MODULE)
 add_library(OBS::mac-dal-plugin ALIAS mac-dal-plugin)
@@ -41,8 +38,9 @@ target_include_directories(
 
 target_compile_options(mac-dal-plugin PRIVATE -fobjc-arc -fobjc-weak)
 
-target_link_libraries(mac-dal-plugin PRIVATE ${COCOA} ${COREMEDIA}
-                                             ${COREMEDIAIO} ${COREVIDEO})
+target_link_libraries(
+  mac-dal-plugin PRIVATE ${COCOA} ${COREMEDIA} ${COREMEDIAIO} ${COREVIDEO}
+                         ${IOSURFACE})
 
 set(MACOSX_PLUGIN_BUNDLE_TYPE "BNDL")
 target_sources(mac-dal-plugin PRIVATE placeholder.png)
@@ -58,6 +56,10 @@ set_target_properties(
              FOLDER "plugins"
              VERSION "0"
              SOVERSION "0"
+             # Force the DAL plugin to be built for arm64e as well. Note that
+             # we cannot build OBS for arm64e, since its libraries are not
+             # built for this architecture at the moment.
+             OSX_ARCHITECTURES "x86_64;arm64;arm64e"
              LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/../../"
              MACOSX_BUNDLE_INFO_PLIST
              "${CMAKE_SOURCE_DIR}/cmake/bundle/macOS/Virtualcam-Info.plist.in")

plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.h

@@ -6,16 +6,16 @@
 //
 
 #import <Foundation/Foundation.h>
+#import <CoreVideo/CoreVideo.h>
 
 NS_ASSUME_NONNULL_BEGIN
 
 @protocol MachClientDelegate
 
-- (void)receivedFrameWithSize:(NSSize)size
-		    timestamp:(uint64_t)timestamp
-		 fpsNumerator:(uint32_t)fpsNumerator
-	       fpsDenominator:(uint32_t)fpsDenominator
-		    frameData:(NSData *)frameData;
+- (void)receivedPixelBuffer:(CVPixelBufferRef)frame
+		  timestamp:(uint64_t)timestamp
+	       fpsNumerator:(uint32_t)fpsNumerator
+	     fpsDenominator:(uint32_t)fpsDenominator;
 - (void)receivedStop;
 
 @end

plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.mm

@@ -101,29 +101,37 @@ - (void)handlePortMessage:(NSPortMessage *)message
 		break;
 	case MachMsgIdFrame:
 		VLog(@"Received frame message");
-		if (components.count >= 6) {
-			CGFloat width;
-			[components[0] getBytes:&width length:sizeof(width)];
-			CGFloat height;
-			[components[1] getBytes:&height length:sizeof(height)];
+		if (components.count >= 4) {
+			NSMachPort *framePort = (NSMachPort *)components[0];
+			IOSurfaceRef surface = IOSurfaceLookupFromMachPort(
+				[framePort machPort]);
+
+			CVPixelBufferRef frame;
+			CVPixelBufferCreateWithIOSurface(kCFAllocatorDefault,
+							 surface, NULL, &frame);
+
 			uint64_t timestamp;
-			[components[2] getBytes:&timestamp
+			[components[1] getBytes:&timestamp
 					 length:sizeof(timestamp)];
-			VLog(@"Received frame data: %fx%f (%llu)", width,
-			     height, timestamp);
-			NSData *frameData = components[3];
+
+			VLog(@"Received frame data: %zux%zu (%llu)",
+			     CVPixelBufferGetWidth(frame),
+			     CVPixelBufferGetHeight(frame), timestamp);
+
 			uint32_t fpsNumerator;
-			[components[4] getBytes:&fpsNumerator
+			[components[2] getBytes:&fpsNumerator
 					 length:sizeof(fpsNumerator)];
 			uint32_t fpsDenominator;
-			[components[5] getBytes:&fpsDenominator
+			[components[3] getBytes:&fpsDenominator
 					 length:sizeof(fpsDenominator)];
-			[self.delegate
-				receivedFrameWithSize:NSMakeSize(width, height)
-					    timestamp:timestamp
-					 fpsNumerator:fpsNumerator
-				       fpsDenominator:fpsDenominator
-					    frameData:frameData];
+
+			[self.delegate receivedPixelBuffer:frame
+						 timestamp:timestamp
+					      fpsNumerator:fpsNumerator
+					    fpsDenominator:fpsDenominator];
+
+			CVPixelBufferRelease(frame);
+			CFRelease(surface);
 		}
 		break;
 	case MachMsgIdStop:

plugins/mac-virtualcam/src/dal-plugin/OBSDALPlugIn.mm

@@ -203,19 +203,21 @@ - (void)setPropertyDataWithAddress:(CMIOObjectPropertyAddress)address
 
 #pragma mark - MachClientDelegate
 
-- (void)receivedFrameWithSize:(NSSize)size
-		    timestamp:(uint64_t)timestamp
-		 fpsNumerator:(uint32_t)fpsNumerator
-	       fpsDenominator:(uint32_t)fpsDenominator
-		    frameData:(NSData *)frameData
+- (void)receivedPixelBuffer:(CVPixelBufferRef)frame
+		  timestamp:(uint64_t)timestamp
+	       fpsNumerator:(uint32_t)fpsNumerator
+	     fpsDenominator:(uint32_t)fpsDenominator
 {
+	size_t width = CVPixelBufferGetWidth(frame);
+	size_t height = CVPixelBufferGetHeight(frame);
+
 	dispatch_sync(_stateQueue, ^{
 		if (_state == PlugInStateWaitingForServer) {
 			NSUserDefaults *defaults =
 				[NSUserDefaults standardUserDefaults];
-			[defaults setInteger:size.width
+			[defaults setInteger:(long)width
 				      forKey:kTestCardWidthKey];
-			[defaults setInteger:size.height
+			[defaults setInteger:(long)height
 				      forKey:kTestCardHeightKey];
 			[defaults setDouble:(double)fpsNumerator /
 					    (double)fpsDenominator
@@ -234,11 +236,10 @@ - (void)receivedFrameWithSize:(NSSize)size
 		dispatch_time(DISPATCH_TIME_NOW, 5.0 * NSEC_PER_SEC),
 		5.0 * NSEC_PER_SEC, (1ull * NSEC_PER_SEC) / 10);
 
-	[self.stream queueFrameWithSize:size
-			      timestamp:timestamp
-			   fpsNumerator:fpsNumerator
-			 fpsDenominator:fpsDenominator
-			      frameData:frameData];
+	[self.stream queuePixelBuffer:frame
+			    timestamp:timestamp
+			 fpsNumerator:fpsNumerator
+		       fpsDenominator:fpsDenominator];
 }
 
 - (void)receivedStop

plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.h

@@ -18,6 +18,7 @@
 //  along with obs-mac-virtualcam. If not, see <http://www.gnu.org/licenses/>.
 
 #import "OBSDALObjectStore.h"
+#import <CoreVideo/CoreVideo.h>
 
 NS_ASSUME_NONNULL_BEGIN
 
@@ -35,11 +36,10 @@ NS_ASSUME_NONNULL_BEGIN
 
 - (void)stopServingDefaultFrames;
 
-- (void)queueFrameWithSize:(NSSize)size
-		 timestamp:(uint64_t)timestamp
-	      fpsNumerator:(uint32_t)fpsNumerator
-	    fpsDenominator:(uint32_t)fpsDenominator
-		 frameData:(NSData *)frameData;
+- (void)queuePixelBuffer:(CVPixelBufferRef)frame
+	       timestamp:(uint64_t)timestamp
+	    fpsNumerator:(uint32_t)fpsNumerator
+	  fpsDenominator:(uint32_t)fpsDenominator;
 
 @end