From 99d1337de2191f4d11957d0036a2a56ea8f9aaf6 Mon Sep 17 00:00:00 2001
From: "Ryan C. Gordon" <[EMAIL REDACTED]>
Date: Tue, 6 Feb 2024 01:19:12 -0500
Subject: [PATCH] camera: Reenabled macOS/iOS support, with rewritten CoreMedia
implementation.
---
include/SDL3/SDL_camera.h | 4 +-
include/build_config/SDL_build_config.h.cmake | 2 -
include/build_config/SDL_build_config_ios.h | 6 +-
include/build_config/SDL_build_config_macos.h | 2 -
src/camera/SDL_camera.c | 10 +-
src/camera/SDL_syscamera.h | 8 +-
src/camera/coremedia/SDL_camera_coremedia.m | 712 ++++++++----------
.../SDL_camera_mediafoundation.c | 2 -
8 files changed, 317 insertions(+), 429 deletions(-)
diff --git a/include/SDL3/SDL_camera.h b/include/SDL3/SDL_camera.h
index a57a66a069967..c070666273d70 100644
--- a/include/SDL3/SDL_camera.h
+++ b/include/SDL3/SDL_camera.h
@@ -66,8 +66,8 @@ typedef struct SDL_CameraSpec
Uint32 format; /**< Frame SDL_PixelFormatEnum format */
int width; /**< Frame width */
int height; /**< Frame height */
- int interval_numerator; /**< Frame rate numerator ((dom / num) == fps) */
- int interval_denominator; /**< Frame rate demoninator ((dom / num) == fps)*/
+ int interval_numerator; /**< Frame rate numerator ((dom / num) == fps, (num / dom) == duration) */
+ int interval_denominator; /**< Frame rate demoninator ((dom / num) == fps, (num / dom) == duration) */
} SDL_CameraSpec;
/**
diff --git a/include/build_config/SDL_build_config.h.cmake b/include/build_config/SDL_build_config.h.cmake
index 11c1ad716ecd0..d25b31b632072 100644
--- a/include/build_config/SDL_build_config.h.cmake
+++ b/include/build_config/SDL_build_config.h.cmake
@@ -246,8 +246,6 @@
#cmakedefine USE_POSIX_SPAWN @USE_POSIX_SPAWN@
-#cmakedefine HAVE_COREMEDIA
-
/* SDL internal assertion support */
#if @SDL_DEFAULT_ASSERT_LEVEL_CONFIGURED@
#cmakedefine SDL_DEFAULT_ASSERT_LEVEL @SDL_DEFAULT_ASSERT_LEVEL@
diff --git a/include/build_config/SDL_build_config_ios.h b/include/build_config/SDL_build_config_ios.h
index 649ff4e86690d..ed3b5480895cd 100644
--- a/include/build_config/SDL_build_config_ios.h
+++ b/include/build_config/SDL_build_config_ios.h
@@ -198,8 +198,6 @@
#define SDL_VIDEO_METAL 1
#endif
-#define HAVE_COREMEDIA 1
-
/* Enable system power support */
#define SDL_POWER_UIKIT 1
@@ -213,6 +211,10 @@
#define SDL_FILESYSTEM_COCOA 1
/* enable camera support */
+#ifndef SDL_PLATFORM_TVOS
#define SDL_CAMERA_DRIVER_COREMEDIA 1
+#endif
+
+#define SDL_CAMERA_DRIVER_DUMMY 1
#endif /* SDL_build_config_ios_h_ */
diff --git a/include/build_config/SDL_build_config_macos.h b/include/build_config/SDL_build_config_macos.h
index a8bed484d94d3..7766ddc0932e8 100644
--- a/include/build_config/SDL_build_config_macos.h
+++ b/include/build_config/SDL_build_config_macos.h
@@ -261,8 +261,6 @@
#endif
#endif
-#define HAVE_COREMEDIA 1
-
/* Enable system power support */
#define SDL_POWER_MACOSX 1
diff --git a/src/camera/SDL_camera.c b/src/camera/SDL_camera.c
index 3a7854a19f31d..6088f3d49d4ac 100644
--- a/src/camera/SDL_camera.c
+++ b/src/camera/SDL_camera.c
@@ -73,6 +73,12 @@ const char *SDL_GetCurrentCameraDriver(void)
return camera_driver.name;
}
+char *SDL_GetCameraThreadName(SDL_CameraDevice *device, char *buf, size_t buflen)
+{
+ (void)SDL_snprintf(buf, buflen, "SDLCamera%d", (int) device->instance_id);
+ return buf;
+}
+
int SDL_AddCameraFormat(CameraFormatAddData *data, Uint32 fmt, int w, int h, int interval_numerator, int interval_denominator)
{
SDL_assert(data != NULL);
@@ -683,7 +689,7 @@ SDL_bool SDL_CameraThreadIterate(SDL_CameraDevice *device)
failed = SDL_TRUE;
}
- // we can let go of the lock once we've tried to grab a frame of video and maybe moved the output frame from the empty to the filled list.
+ // we can let go of the lock once we've tried to grab a frame of video and maybe moved the output frame off the empty list.
// this lets us chew up the CPU for conversion and scaling without blocking other threads.
SDL_UnlockMutex(device->lock);
@@ -988,7 +994,7 @@ SDL_Camera *SDL_OpenCameraDevice(SDL_CameraDeviceID instance_id, const SDL_Camer
// Start the camera thread if necessary
if (!camera_driver.impl.ProvidesOwnCallbackThread) {
char threadname[64];
- SDL_snprintf(threadname, sizeof (threadname), "SDLCamera%d", (int) instance_id);
+ SDL_GetCameraThreadName(device, threadname, sizeof (threadname));
device->thread = SDL_CreateThreadInternal(CameraThread, threadname, 0, device);
if (!device->thread) {
ClosePhysicalCameraDevice(device);
diff --git a/src/camera/SDL_syscamera.h b/src/camera/SDL_syscamera.h
index b99239ec51d72..9f556523d109a 100644
--- a/src/camera/SDL_syscamera.h
+++ b/src/camera/SDL_syscamera.h
@@ -28,10 +28,7 @@
#define DEBUG_CAMERA 0
-// !!! FIXME: update these drivers!
-#ifdef SDL_CAMERA_DRIVER_COREMEDIA
-#undef SDL_CAMERA_DRIVER_COREMEDIA
-#endif
+// !!! FIXME: update this driver!
#ifdef SDL_CAMERA_DRIVER_ANDROID
#undef SDL_CAMERA_DRIVER_ANDROID
#endif
@@ -53,6 +50,9 @@ extern SDL_CameraDevice *SDL_FindPhysicalCameraDeviceByCallback(SDL_bool (*callb
// Backends should call this when the user has approved/denied access to a camera.
extern void SDL_CameraDevicePermissionOutcome(SDL_CameraDevice *device, SDL_bool approved);
+// Backends can call this to get a standardized name for a thread to power a specific camera device.
+extern char *SDL_GetCameraThreadName(SDL_CameraDevice *device, char *buf, size_t buflen);
+
// These functions are the heart of the camera threads. Backends can call them directly if they aren't using the SDL-provided thread.
extern void SDL_CameraThreadSetup(SDL_CameraDevice *device);
extern SDL_bool SDL_CameraThreadIterate(SDL_CameraDevice *device);
diff --git a/src/camera/coremedia/SDL_camera_coremedia.m b/src/camera/coremedia/SDL_camera_coremedia.m
index c37962026c205..cf07d47cce7a6 100644
--- a/src/camera/coremedia/SDL_camera_coremedia.m
+++ b/src/camera/coremedia/SDL_camera_coremedia.m
@@ -26,17 +26,6 @@
#include "../SDL_camera_c.h"
#include "../../thread/SDL_systhread.h"
-#if defined(HAVE_COREMEDIA) && defined(SDL_PLATFORM_MACOS) && (__MAC_OS_X_VERSION_MAX_ALLOWED < 101500)
-// AVCaptureDeviceTypeBuiltInWideAngleCamera requires macOS SDK 10.15
-#undef HAVE_COREMEDIA
-#endif
-
-#ifdef SDL_PLATFORM_TVOS
-#undef HAVE_COREMEDIA
-#endif
-
-#ifdef HAVE_COREMEDIA
-
#import <AVFoundation/AVFoundation.h>
#import <CoreMedia/CoreMedia.h>
@@ -50,537 +39,434 @@
* MACOSX:
* Add to the Code Sign Entitlement file:
* <key>com.apple.security.device.camera</key> <true/>
- *
- *
- * IOS:
- *
- * - Need to link with:: CoreMedia CoreVideo
- * - Add #define SDL_CAMERA 1
- * to SDL_build_config_ios.h
*/
-@class MySampleBufferDelegate;
-
-struct SDL_PrivateCameraData
-{
- dispatch_queue_t queue;
- MySampleBufferDelegate *delegate;
- AVCaptureSession *session;
- CMSimpleQueueRef frame_queue;
-};
-
-static NSString *fourcc_to_nstring(Uint32 code)
+static Uint32 CoreMediaFormatToSDL(FourCharCode fmt)
{
- Uint8 buf[4];
- *(Uint32 *)buf = code;
- return [NSString stringWithFormat:@"%c%c%c%c", buf[3], buf[2], buf[1], buf[0]];
+ switch (fmt) {
+ #define CASE(x, y) case x: return y
+ // the 16LE ones should use 16BE if we're on a Bigendian system like PowerPC,
+ // but at current time there is no bigendian Apple platform that has CoreMedia.
+ CASE(kCMPixelFormat_16LE555, SDL_PIXELFORMAT_RGB555);
+ CASE(kCMPixelFormat_16LE5551, SDL_PIXELFORMAT_RGBA5551);
+ CASE(kCMPixelFormat_16LE565, SDL_PIXELFORMAT_RGB565);
+ CASE(kCMPixelFormat_24RGB, SDL_PIXELFORMAT_RGB24);
+ CASE(kCMPixelFormat_32ARGB, SDL_PIXELFORMAT_ARGB32);
+ CASE(kCMPixelFormat_32BGRA, SDL_PIXELFORMAT_BGRA32);
+ CASE(kCMPixelFormat_422YpCbCr8, SDL_PIXELFORMAT_YUY2);
+ CASE(kCMPixelFormat_422YpCbCr8_yuvs, SDL_PIXELFORMAT_UYVY);
+ #undef CASE
+ default:
+ #if DEBUG_CAMERA
+ SDL_Log("CAMERA: Unknown format FourCharCode '%d'", (int) fmt);
+ #endif
+ break;
+ }
+ return SDL_PIXELFORMAT_UNKNOWN;
}
-static NSArray<AVCaptureDevice *> *DiscoverCameraDevices()
-{
- NSArray *deviceType = @[AVCaptureDeviceTypeBuiltInWideAngleCamera];
+@class SDLCaptureVideoDataOutputSampleBufferDelegate;
- AVCaptureDeviceDiscoverySession *discoverySession = [AVCaptureDeviceDiscoverySession
- discoverySessionWithDeviceTypes:deviceType
- mediaType:AVMediaTypeVideo
- position:AVCaptureDevicePositionUnspecified];
+// just a simple wrapper to help ARC manage memory...
+@interface SDLPrivateCameraData : NSObject
+@property(nonatomic, retain) AVCaptureSession *session;
+@property(nonatomic, retain) SDLCaptureVideoDataOutputSampleBufferDelegate *delegate;
+@property(nonatomic, assign) CMSampleBufferRef current_sample;
+@end
- NSArray<AVCaptureDevice *> *devices = discoverySession.devices;
+@implementation SDLPrivateCameraData
+@end
- if ([devices count] > 0) {
- return devices;
- } else {
- AVCaptureDevice *captureDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
- if (captureDevice == nil) {
- return devices;
+
+static SDL_bool CheckCameraPermissions(SDL_CameraDevice *device)
+{
+ if (device->permission == 0) { // still expecting a permission result.
+ if (@available(macOS 14, *)) {
+ const AVAuthorizationStatus status = [AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeVideo];
+ if (status != AVAuthorizationStatusNotDetermined) { // NotDetermined == still waiting for an answer from the user.
+ SDL_CameraDevicePermissionOutcome(device, (status == AVAuthorizationStatusAuthorized) ? SDL_TRUE : SDL_FALSE);
+ }
} else {
- NSArray<AVCaptureDevice *> *default_device = @[ captureDevice ];
- return default_device;
+ SDL_CameraDevicePermissionOutcome(device, SDL_TRUE); // always allowed (or just unqueryable...?) on older macOS.
}
}
- return devices;
+ return (device->permission > 0);
}
-static AVCaptureDevice *GetCameraDeviceByName(const char *dev_name)
-{
- NSArray<AVCaptureDevice *> *devices = DiscoverCameraDevices();
+// this delegate just receives new video frames on a Grand Central Dispatch queue, and fires off the
+// main device thread iterate function directly to consume it.
+@interface SDLCaptureVideoDataOutputSampleBufferDelegate : NSObject<AVCaptureVideoDataOutputSampleBufferDelegate>
+ @property SDL_CameraDevice *device;
+ -(id) init:(SDL_CameraDevice *) dev;
+ -(void) captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection;
+@end
- for (AVCaptureDevice *device in devices) {
- char buf[1024];
- NSString *cameraID = [device localizedName];
- const char *str = [cameraID UTF8String];
- SDL_snprintf(buf, sizeof (buf) - 1, "%s", str);
- if (SDL_strcmp(buf, dev_name) == 0) {
- return device;
+@implementation SDLCaptureVideoDataOutputSampleBufferDelegate
+
+ -(id) init:(SDL_CameraDevice *) dev {
+ if ( self = [super init] ) {
+ _device = dev;
}
+ return self;
}
- return nil;
-}
-static Uint32 nsfourcc_to_sdlformat(NSString *nsfourcc)
-{
- const char *str = [nsfourcc UTF8String];
-
- /* FIXME
- * on IOS this mode gives 2 planes, and it's NV12
- * on macos, 1 plane/ YVYU
- */
- #ifdef SDL_PLATFORM_MACOS
- if (SDL_strcmp("420v", str) == 0) return SDL_PIXELFORMAT_YVYU;
- #else
- if (SDL_strcmp("420v", str) == 0) return SDL_PIXELFORMAT_NV12;
- #endif
+ - (void) captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection
+ {
+ SDL_CameraDevice *device = self.device;
+ if (!device || !device->hidden) {
+ return; // oh well.
+ }
- if (SDL_strcmp("yuvs", str) == 0) return SDL_PIXELFORMAT_UYVY;
- if (SDL_strcmp("420f", str) == 0) return SDL_PIXELFORMAT_UNKNOWN;
+ if (!CheckCameraPermissions(device)) {
+ return; // nothing to do right now, dump what is probably a completely black frame.
+ }
- #if DEBUG_CAMERA
- SDL_Log("CAMERA: Unknown format '%s'", str);
- #endif
+ SDLPrivateCameraData *hidden = (__bridge SDLPrivateCameraData *) device->hidden;
+ hidden.current_sample = sampleBuffer;
+ SDL_CameraThreadIterate(device);
+ hidden.current_sample = NULL;
+ }
- return SDL_PIXELFORMAT_UNKNOWN;
-}
+ - (void)captureOutput:(AVCaptureOutput *)output didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection
+ {
+ #if DEBUG_CAMERA
+ SDL_Log("CAMERA: Drop frame.");
+ #endif
+ }
+@end
-static NSString *sdlformat_to_nsfourcc(Uint32 fmt)
+static int COREMEDIA_WaitDevice(SDL_CameraDevice *device)
{
- const char *str = "";
- NSString *result;
-
-#ifdef SDL_PLATFORM_MACOS
- if (fmt == SDL_PIXELFORMAT_YVYU) str = "420v";
-#else
- if (fmt == SDL_PIXELFORMAT_NV12) str = "420v";
-#endif
- if (fmt == SDL_PIXELFORMAT_UYVY) str = "yuvs";
-
- return [[NSString alloc] initWithUTF8String: str];
+ return 0; // this isn't used atm, since we run our own thread out of Grand Central Dispatch.
}
+static int COREMEDIA_AcquireFrame(SDL_CameraDevice *device, SDL_Surface *frame, Uint64 *timestampNS)
+{
+ int retval = 1;
+ SDLPrivateCameraData *hidden = (__bridge SDLPrivateCameraData *) device->hidden;
+ CMSampleBufferRef sample_buffer = hidden.current_sample;
+ hidden.current_sample = NULL;
+ SDL_assert(sample_buffer != NULL); // should only have been called from our delegate with a new frame.
+
+ CMSampleTimingInfo timinginfo;
+ if (CMSampleBufferGetSampleTimingInfo(sample_buffer, 0, &timinginfo) == noErr) {
+ *timestampNS = (Uint64) (CMTimeGetSeconds(timinginfo.presentationTimeStamp) * ((Float64) SDL_NS_PER_SECOND));
+ } else {
+ SDL_assert(!"this shouldn't happen, I think.");
+ *timestampNS = 0;
+ }
-@interface MySampleBufferDelegate : NSObject<AVCaptureVideoDataOutputSampleBufferDelegate>
- @property struct SDL_PrivateCameraData *hidden;
- - (void) set: (struct SDL_PrivateCameraData *) val;
-@end
+ CVImageBufferRef image = CMSampleBufferGetImageBuffer(sample_buffer); // does not retain `image` (and we don't want it to).
+ const int numPlanes = (int) CVPixelBufferGetPlaneCount(image);
+ const int planar = (int) CVPixelBufferIsPlanar(image);
-@implementation MySampleBufferDelegate
+ #if DEBUG_CAMERA
+ const int w = (int) CVPixelBufferGetWidth(image);
+ const int h = (int) CVPixelBufferGetHeight(image);
+ const int sz = (int) CVPixelBufferGetDataSize(image);
+ const int pitch = (int) CVPixelBufferGetBytesPerRow(image);
+ SDL_Log("CAMERA: buffer planar=%d numPlanes=%d %d x %d sz=%d pitch=%d", planar, numPlanes, w, h, sz, pitch);
+ #endif
- - (void) set: (struct SDL_PrivateCameraData *) val {
- _hidden = val;
- }
+ // !!! FIXME: this currently copies the data to the surface (see FIXME about non-contiguous planar surfaces, but in theory we could just keep this locked until ReleaseFrame...
+ CVPixelBufferLockBaseAddress(image, 0);
- - (void) captureOutput:(AVCaptureOutput *)output
- didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
- fromConnection:(AVCaptureConnection *) connection {
- CFRetain(sampleBuffer);
- CMSimpleQueueEnqueue(_hidden->frame_queue, sampleBuffer);
+ if ((planar == 0) && (numPlanes == 0)) {
+ const int pitch = (int) CVPixelBufferGetBytesPerRow(image);
+ const size_t buflen = pitch * frame->h;
+ frame->pixels = SDL_aligned_alloc(SDL_SIMDGetAlignment(), buflen);
+ if (frame->pixels == NULL) {
+ retval = -1;
+ } else {
+ frame->pitch = pitch;
+ SDL_memcpy(frame->pixels, CVPixelBufferGetBaseAddress(image), buflen);
}
-
- - (void)captureOutput:(AVCaptureOutput *)output
- didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer
- fromConnection:(AVCaptureConnection *)connection {
- #if DEBUG_CAMERA
- SDL_Log("CAMERA: Drop frame..");
- #endif
+ } else {
+ // !!! FIXME: we have an open issue in SDL3 to allow SDL_Surface to support non-contiguous planar data, but we don't have it yet.
+ size_t buflen = 0;
+ for (int i = 0; (i < numPlanes) && (i < 3); i++) {
+ buflen += CVPixelBufferGetBytesPerRowOfPlane(image, i);
}
-@end
+ buflen *= frame->h;
-static int COREMEDIA_OpenDevice(SDL_CameraDevice *_this)
-{
- _this->hidden = (struct SDL_PrivateCameraData *) SDL_calloc(1, sizeof (struct SDL_PrivateCameraData));
- if (_this->hidden == NULL) {
- return -1;
+ frame->pixels = SDL_aligned_alloc(SDL_SIMDGetAlignment(), buflen);
+ if (frame->pixels == NULL) {
+ retval = -1;
+ } else {
+ Uint8 *dst = frame->pixels;
+ frame->pitch = (int) CVPixelBufferGetBytesPerRowOfPlane(image, 0); // this is what SDL3 currently expects, probably incorrectly.
+ for (int i = 0; (i < numPlanes) && (i < 3); i++) {
+ const void *src = CVPixelBufferGetBaseAddressOfPlane(image, i);
+ const size_t pitch = CVPixelBufferGetBytesPerRowOfPlane(image, i);
+ SDL_memcpy(dst, src, pitch * frame->h);
+ dst += pitch * frame->h;
+ }
+ }
}
- return 0;
+
+ CVPixelBufferUnlockBaseAddress(image, 0);
+
+ return retval;
}
-static void COREMEDIA_CloseDevice(SDL_CameraDevice *_this)
+static void COREMEDIA_ReleaseFrame(SDL_CameraDevice *device, SDL_Surface *frame)
{
- if (!_this) {
- return;
- }
+ // !!! FIXME: this currently copies the data to the surface, but in theory we could just keep this locked until ReleaseFrame...
+ SDL_aligned_free(frame->pixels);
+}
- if (_this->hidden) {
- AVCaptureSession *session = _this->hidden->session;
+static void COREMEDIA_CloseDevice(SDL_CameraDevice *device)
+{
+ if (device && device->hidden) {
+ SDLPrivateCameraData *hidden = (SDLPrivateCameraData *) CFBridgingRelease(device->hidden);
+ device->hidden = NULL;
+ AVCaptureSession *session = hidden.session;
if (session) {
- AVCaptureInput *input;
- AVCaptureVideoDataOutput *output;
- input = [session.inputs objectAtIndex:0];
- [session removeInput:input];
- output = (AVCaptureVideoDataOutput*)[session.outputs objectAtIndex:0];
- [session removeOutput:output];
- // TODO more cleanup ?
+ hidden.session = nil;
+ [session stopRunning];
+ [session removeInput:[session.inputs objectAtIndex:0]];
+ [session removeOutput:(AVCaptureVideoDataOutput*)[session.outputs objectAtIndex:0]];
+ session = nil;
}
- if (_this->hidden->frame_queue) {
- CFRelease(_this->hidden->frame_queue);
- }
-
- SDL_free(_this->hidden);
- _this->hidden = NULL;
+ hidden.delegate = NULL;
+ hidden.current_sample = NULL;
}
}
-static int COREMEDIA_InitDevice(SDL_CameraDevice *_this)
+static int COREMEDIA_OpenDevice(SDL_CameraDevice *device, const SDL_CameraSpec *spec)
{
- // !!! FIXME: autorelease pool?
- NSString *fmt = sdlformat_to_nsfourcc(_this->spec.format);
- int w = _this->spec.width;
- int h = _this->spec.height;
-
- NSError *error = nil;
- AVCaptureDevice *device = nil;
- AVCaptureDeviceInput *input = nil;
- AVCaptureVideoDataOutput *output = nil;
-
- AVCaptureDeviceFormat *spec_format = nil;
-
-#ifdef SDL_PLATFORM_MACOS
- if (@available(macOS 10.15, *)) {
- // good.
- } else {
- return -1;
- }
-#endif
-
- device = GetCameraDeviceByName(_this->dev_name);
- if (!device) {
- goto error;
- }
-
- _this->hidden->session = [[AVCaptureSession alloc] init];
- if (_this->hidden->session == nil) {
- goto error;
- }
-
- [_this->hidden->session setSessionPreset:AVCaptureSessionPresetHigh];
+ AVCaptureDevice *avdevice = (__bridge AVCaptureDevice *) device->handle;
// Pick format that matches the spec
- NSArray<AVCaptureDeviceFormat *> *formats = [device formats];
+ const Uint32 sdlfmt = spec->format;
+ const int w = spec->width;
+ const int h = spec->height;
+ const int rate = spec->interval_denominator;
+ AVCaptureDeviceFormat *spec_format = nil;
+ NSArray<AVCaptureDeviceFormat *> *formats = [avdevice formats];
for (AVCaptureDeviceFormat *format in formats) {
CMFormatDescriptionRef formatDescription = [format formatDescription];
- FourCharCode mediaSubType = CMFormatDescriptionGetMediaSubType(formatDescription);
- NSString *str = fourcc_to_nstring(mediaSubType);
- if ([str isEqualToString:fmt]) {
- CMVideoDimensions dim = CMVideoFormatDescriptionGetDimensions(formatDescription);
- if (dim.width == w && dim.height == h) {
- spec_format = format;
- break;
- }
+ if (CoreMediaFormatToSDL(CMFormatDescriptionGetMediaSubType(formatDescription)) != sdlfmt) {
+ continue;
+ }
+
+ const CMVideoDimensions dim = CMVideoFormatDescriptionGetDimensions(formatDescription);
+ if ( ((int) dim.width != w) || (((int) dim.height) != h) ) {
+ continue;
+ }
+
+ for (AVFrameRateRange *framerate in format.videoSupportedFrameRateRanges) {
+ if ((rate == (int) SDL_ceil((double) framerate.minFrameRate)) || (rate == (int) SDL_floor((double) framerate.maxFrameRate))) {
+ spec_format = format;
+ break;
}
}
+
+ if (spec_format != nil) {
+ break;
+ }
}
if (spec_format == nil) {
- return SDL_SetError("format not found");
+ return SDL_SetError("camera spec format not available");
+ } else if (![avdevice lockForConfiguration:NULL]) {
+ return SDL_SetError("Cannot lockForConfiguration");
}
- // Set format
- if ([device lockForConfiguration:NULL] == YES) {
- device.activeFormat = spec_format;
- [device unlockForConfiguration];
- } else {
- return SDL_SetError("Cannot lockForConfiguration");
+ avdevice.activeFormat = spec_format;
+ [avdevice unlockForConfiguration];
+
+ AVCaptureSession *session = [[AVCaptureSession alloc] init];
+ if (session == nil) {
+ return SDL_SetError("Failed to allocate/init AVCaptureSession");
}
- // Input
- input = [AVCaptureDeviceInput deviceInputWithDevice:device error:&error];
+ session.sessionPreset = AVCaptureSessionPresetHigh;
+
+ NSError *error = nil;
+ AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:avdevice error:&error];
if (!input) {
return SDL_SetError("Cannot create AVCaptureDeviceInput");
}
- // Output
- output = [[AVCaptureVideoDataOutput alloc] init];
-
-#ifdef SDL_PLATFORM_MACOS
- // FIXME this now fail on ios ... but not using anything works...
-
- // Specify the pixel format
- output.videoSettings =
- [NSDictionary dictionaryWithObject:
- [NSNumber numberWithInt:kCVPixelFormatType_422YpCbCr8]
- forKey:(id)kCVPixelBufferPixelFormatTypeKey];
-#endif
-
- _this->hidden->delegate = [[MySampleBufferDelegate alloc] init];
- [_this->hidden->delegate set:_this->hidden];
-
+ AVCaptureVideoDataOutput *output = [[AVCaptureVideoDataOutput alloc] init];
+ if (!output) {
+ return SDL_SetError("Cannot create AVCaptureVideoDataOutput");
+ }
- CMSimpleQueueCreate(kCFAllocatorDefault, 30 /* buffers */, &_this->hidden->frame_queue);
- if (_this->hidden->frame_queue == nil) {
- return SDL_SetError("CMSimpleQueueCreate() failed");
+ char threadname[64];
+ SDL_GetCameraThreadName(device, threadname, sizeof (threadname));
+ dispatch_queue_t queue = dispatch_queue_create(threadname, NULL);
+ //dispatch_queue_t queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0);
+ if (!queue) {
+ return SDL_SetError("dispatch_queue_create() failed");
}
- _this->hidden->queue = dispatch_queue_create("my_queue", NULL);
- [output setSampleBufferDelegate:_this->hidden->delegate queue:_this->hidden->queue];
+ SDLCaptureVideoDataOutputSampleBufferDelegate *delegate = [[SDLCaptureVideoDataOutputSampleBufferDelegate alloc] init:device];
+ if (delegate == nil) {
+ return SDL_SetError("Cannot create SDLCaptureVideoDataOutputSampleBufferDelegate");
+ }
+ [output setSampleBufferDelegate:delegate queue:queue];
- if ([_this->hidden->session canAddInput:input] ){
- [_this->hidden->session addInput:input];
- } else {
+ if (![session canAddInput:input]) {
return SDL_SetError("Cannot add AVCaptureDeviceInput");
}
+ [session addInput:input];
- if ([_this->hidden->session canAddOutput:output] ){
- [_this->hidden->session addOutput:output];
- } else {
+ if (![session canAddOutput:output]) {
return SDL_SetError("Cannot add AVCaptureVideoDataOutput");
}
+ [session addOutput:output];
- [_this->hidden->session commitConfiguration];
-
- return 0;
-}
+ [session commitConfiguration];
-static int COREMEDIA_GetDeviceSpec(SDL_CameraDevice *_this, SDL_CameraSpec *spec)
-{
- // !!! FIXME: make sure higher level checks spec != NULL
- if (spec) {
- SDL_copyp(spec, &_this->spec);
- return 0;
+ SDLPrivateCameraData *hidden = [[SDLPrivateCameraData alloc] init];
+ if (hidden == nil) {
+ return SDL_SetError("Cannot create SDLPrivateCameraData");
}
- return -1;
-}
-static int COREMEDIA_StartCamera(SDL_CameraDevice *_this)
-{
- [_this->hidden->session startRunning];
- return 0;
-}
+ hidden.session = session;
+ hidden.delegate = delegate;
+ hidden.current_sample = NULL;
+ device->hidden = (struct SDL_PrivateCameraData *)CFBridgingRetain(hidden);
-static int COREMEDIA_StopCamera(SDL_CameraDevice *_this)
-{
- [_this->hidden->session stopRunning];
- return 0;
-}
+ [session startRunning]; // !!! FIXME: docs say this can block while camera warms up and shouldn't be done on main thread. Maybe push through `queue`?
-static int COREMEDIA_AcquireFrame(SDL_CameraDevice *_this, SDL_CameraFrame *frame)
-{
- if (CMSimpleQueueGetCount(_this->hidden->frame_queue) > 0) {
- CMSampleBufferRef sampleBuffer = (CMSampleBufferRef)CMSimpleQueueDequeue(_this->hidden->frame_queue);
- frame->internal = (void *) sampleBuffer;
- frame->timestampNS = SDL_GetTicksNS();
-
- CVImageBufferRef image = CMSampleBufferGetImageBuffer(sampleBuffer);
- const int numPlanes = CVPixelBufferGetPlaneCount(image);
- const int planar = CVPixelBufferIsPlanar(image);
-
- #if DEBUG_CAMERA
- const int w = CVPixelBufferGetWidth(image);
- const int h = CVPixelBufferGetHeight(image);
- const int sz = CVPixelBufferGetDataSize(image);
- const int pitch = CVPixelBufferGetBytesPerRow(image);
- SDL_Log("CAMERA: buffer planar=%d count:%d %d x %d sz=%d pitch=%d", planar, numPlanes, w, h, sz, pitch);
- #endif
+ CheckCameraPermissions(device); // check right away, in case the process is already granted permission.
- CVPixelBufferLockBaseAddress(image, 0);
-
- if ((planar == 0) && (numPlanes == 0)) {
- frame->pitch[0] = CVPixelBufferGetBytesPerRow(image);
- frame->data[0] = CVPixelBufferGetBaseAddress(image);
- frame->num_planes = 1;
- } else {
- for (int i = 0; (i < numPlanes) && (i < 3); i++) {
- frame->num_planes += 1;
- frame->data[i] = CVPixelBufferGetBaseAddressOfPlane(image, i);
- frame->pitch[i] = CVPixelBufferGetBytesPerRowOfPlane(image, i);
- }
- }
-
- // Unlocked when frame is released
- } else {
- // no frame
- SDL_Delay(20); // TODO fix some delay
- }
return 0;
}
-static int COREMEDIA_ReleaseFrame(SDL_CameraDevice *_this, SDL_CameraFrame *frame)
+static void COREMEDIA_FreeDeviceHandle(SDL_CameraDevice *device)
{
- if (frame->internal) {
- CMSampleBufferRef sampleBuffer = (CMSampleBufferRef) frame->internal;
- CVImageBufferRef image = CMSampleBufferGetImageBuffer(sampleBuffer);
- CVPixelBufferUnlockBaseAddress(image, 0);
- CFRelease(sampleBuffer);
+ if (device && device->handle) {
+ CFBridgingRelease(device->handle);
}
-
- return 0;
}
-static int COREMEDIA_GetNumFormats(SDL_CameraDevice *_this)
+static void GatherCameraSpecs(AVCaptureDevice *device, CameraFormatAddData *add_data)
{
- AVCaptureDevice *device = GetCameraDeviceByName(_this->dev_name);
- if (device) {
- // LIST FORMATS
- NSMutableOrderedSet<NSString *> *array_formats = [NSMutableOrderedSet new];
- NSArray<AVCaptureDeviceFormat *> *formats = [device formats];
- for (AVCaptureDeviceFormat *format in formats) {
- // NSLog(@"%@", formats);
- CMFormatDescriptionRef formatDescription = [format formatDescription];
- //NSLog(@"%@", formatDescription);
- FourCharCode mediaSubType = CMFormatDescriptionGetMediaSubType(formatDescription);
- NSString *str = fourcc_to_nstring(mediaSubType);
- [array_formats addObject:str];
- }
- return [array_formats count];
- }
- return 0;
-}
+ SDL_zerop(add_data);
-static int COREMEDIA_GetFormat(SDL_CameraDevice *_this, int index, Uint32 *format)
-{
- AVCaptureDevice *device = GetCameraDeviceByName(_this->dev_name);
- if (device) {
- // LIST FORMATS
- NSMutableOrderedSet<NSString *> *array_formats = [NSMutableOrderedSet new];
- NSArray<AVCaptureDeviceFormat *> *formats = [device formats];
- NSString *str;
-
- for (AVCaptureDeviceFormat *f in formats) {
- FourCharCode mediaSubType;
- CMFormatDescriptionRef formatDescription;
-
- formatDescription = [f formatDescription];
- mediaSubType = CMFormatDescriptionGetMediaSubType(formatDescription);
- str = fourcc_to_nstring(mediaSubType);
- [array_formats addObject:str];
+ for (AVCaptureDeviceFormat *fmt in device.formats) {
+ if (CMFormatDescriptionGetMediaType(fmt.formatDescription) != kCMMediaType_Video) {
+ continue;
}
- str = array_formats[index];
- *format = nsfourcc_to_sdlformat(str);
+ const Uint32 sdlfmt = CoreMediaFormatToSDL(CMFormatDescriptionGetMediaSubType(fmt.formatDescription));
+ if (sdlfmt == SDL_PIXELFORMAT_UNKNOWN) {
+ continue;
+ }
- return 0;
- }
- return -1;
-}
+ const CMVideoDimensions dims = CMVideoFormatDescriptionGetDimensions(fmt
(Patch may be truncated, please check the link at the top of this post.)