From d830cd140b6e5701a9d90b7b0fe10ed292baf46e Mon Sep 17 00:00:00 2001
From: Sam Lantinga <[EMAIL REDACTED]>
Date: Mon, 9 Oct 2023 17:55:06 -0700
Subject: [PATCH] Added support for 0-copy decode and display using Apple
VideoToolbox
---
test/CMakeLists.txt | 7 +-
test/testffmpeg.c | 61 ++++++++++++--
test/testffmpeg_videotoolbox.h | 15 ++++
test/testffmpeg_videotoolbox.m | 147 +++++++++++++++++++++++++++++++++
4 files changed, 221 insertions(+), 9 deletions(-)
create mode 100644 test/testffmpeg_videotoolbox.h
create mode 100644 test/testffmpeg_videotoolbox.m
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 62f0eb7c3b8d..9ca6ab796b26 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -184,7 +184,12 @@ endif()
include("${SDL3_SOURCE_DIR}/cmake/FindFFmpeg.cmake")
if(FFmpeg_FOUND AND FFmpeg_AVCODEC_VERSION VERSION_GREATER_EQUAL "60")
- add_sdl_test_executable(testffmpeg NO_C90 SOURCES testffmpeg.c ${icon_bmp_header})
+ if(APPLE)
+ add_sdl_test_executable(testffmpeg NO_C90 SOURCES testffmpeg.c testffmpeg_videotoolbox.m ${icon_bmp_header})
+ target_link_options(testffmpeg PRIVATE "-Wl,-framework,CoreFoundation" "-Wl,-framework,CoreVideo" "-Wl,-framework,Metal")
+ else()
+ add_sdl_test_executable(testffmpeg NO_C90 SOURCES testffmpeg.c ${icon_bmp_header})
+ endif()
target_link_libraries(testffmpeg PRIVATE ${FFMPEG_LIBRARIES})
if(HAVE_OPENGLES_V2)
#message(STATUS "Enabling EGL support in testffmpeg")
diff --git a/test/testffmpeg.c b/test/testffmpeg.c
index 301a819fbb0e..5e2dbfc1d12f 100644
--- a/test/testffmpeg.c
+++ b/test/testffmpeg.c
@@ -36,6 +36,10 @@
#endif
#endif
+#ifdef __APPLE__
+#include "testffmpeg_videotoolbox.h"
+#endif
+
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/avutil.h>
@@ -64,6 +68,9 @@ static SDL_bool has_EGL_EXT_image_dma_buf_import;
static PFNGLACTIVETEXTUREARBPROC glActiveTextureARBFunc;
static PFNGLEGLIMAGETARGETTEXTURE2DOESPROC glEGLImageTargetTexture2DOESFunc;
#endif
+#ifdef __APPLE__
+static SDL_bool has_videotoolbox_output;
+#endif
static int done;
static SDL_bool CreateWindow(Uint32 window_flags, SDL_bool useEGL)
@@ -87,7 +94,7 @@ static SDL_bool CreateWindow(Uint32 window_flags, SDL_bool useEGL)
SDL_GL_SetAttribute(SDL_GL_GREEN_SIZE, 6);
SDL_GL_SetAttribute(SDL_GL_BLUE_SIZE, 5);
- if (SDL_CreateWindowAndRenderer(WINDOW_WIDTH, WINDOW_HEIGHT, SDL_WINDOW_HIDDEN, &window, &renderer) < 0) {
+ if (SDL_CreateWindowAndRenderer(WINDOW_WIDTH, WINDOW_HEIGHT, window_flags, &window, &renderer) < 0) {
return SDL_FALSE;
}
@@ -116,6 +123,10 @@ static SDL_bool CreateWindow(Uint32 window_flags, SDL_bool useEGL)
}
#endif /* HAVE_EGL */
+#ifdef __APPLE__
+ has_videotoolbox_output = SetupVideoToolboxOutput(renderer);
+#endif
+
return SDL_TRUE;
}
@@ -222,6 +233,11 @@ static SDL_bool SupportedPixelFormat(enum AVPixelFormat format)
(format == AV_PIX_FMT_VAAPI || format == AV_PIX_FMT_DRM_PRIME)) {
return SDL_TRUE;
}
+#ifdef __APPLE__
+ if (has_videotoolbox_output && format == AV_PIX_FMT_VIDEOTOOLBOX) {
+ return SDL_TRUE;
+ }
+#endif
if (GetTextureFormat(format) != SDL_PIXELFORMAT_UNKNOWN) {
return SDL_TRUE;
@@ -474,11 +490,41 @@ static SDL_bool GetTextureForFrame(AVFrame *frame, SDL_Texture **texture)
}
}
-static void HandleVideoFrame(AVFrame *frame, double pts)
+static void DisplayVideoTexture(AVFrame *frame)
{
/* Update the video texture */
GetTextureForFrame(frame, &video_texture);
+ if (frame->linesize[0] < 0) {
+ SDL_RenderTextureRotated(renderer, video_texture, NULL, NULL, 0.0, NULL, SDL_FLIP_VERTICAL);
+ } else {
+ SDL_RenderTexture(renderer, video_texture, NULL, NULL);
+ }
+}
+
+static void DisplayVideoToolbox(AVFrame *frame)
+{
+#ifdef __APPLE__
+ SDL_Rect viewport;
+ SDL_GetRenderViewport(renderer, &viewport);
+ DisplayVideoToolboxFrame(renderer, frame->data[3], 0, 0, frame->width, frame->height, viewport.x, viewport.y, viewport.w, viewport.h);
+#endif
+}
+
+static void DisplayVideoFrame(AVFrame *frame)
+{
+ switch (frame->format) {
+ case AV_PIX_FMT_VIDEOTOOLBOX:
+ DisplayVideoToolbox(frame);
+ break;
+ default:
+ DisplayVideoTexture(frame);
+ break;
+ }
+}
+
+static void HandleVideoFrame(AVFrame *frame, double pts)
+{
/* Quick and dirty PTS handling */
if (!video_start) {
video_start = SDL_GetTicks();
@@ -489,11 +535,7 @@ static void HandleVideoFrame(AVFrame *frame, double pts)
now = (double)(SDL_GetTicks() - video_start) / 1000.0;
}
- if (frame->linesize[0] < 0) {
- SDL_RenderTextureRotated(renderer, video_texture, NULL, NULL, 0.0, NULL, SDL_FLIP_VERTICAL);
- } else {
- SDL_RenderTexture(renderer, video_texture, NULL, NULL);
- }
+ DisplayVideoFrame(frame);
/* Render any bouncing balls */
MoveSprite();
@@ -658,7 +700,7 @@ int main(int argc, char *argv[])
goto quit;
}
- window_flags = SDL_WINDOW_HIDDEN;
+ window_flags = SDL_WINDOW_HIDDEN | SDL_WINDOW_HIGH_PIXEL_DENSITY;
#ifdef __APPLE__
window_flags |= SDL_WINDOW_METAL;
#elif !defined(__WIN32__)
@@ -832,6 +874,9 @@ int main(int argc, char *argv[])
}
return_code = 0;
quit:
+#ifdef __APPLE__
+ CleanupVideoToolboxOutput();
+#endif
SDL_free(positions);
SDL_free(velocities);
av_frame_free(&frame);
diff --git a/test/testffmpeg_videotoolbox.h b/test/testffmpeg_videotoolbox.h
new file mode 100644
index 000000000000..21242955cbca
--- /dev/null
+++ b/test/testffmpeg_videotoolbox.h
@@ -0,0 +1,15 @@
+/*
+ Copyright (C) 1997-2023 Sam Lantinga <slouken@libsdl.org>
+
+ This software is provided 'as-is', without any express or implied
+ warranty. In no event will the authors be held liable for any damages
+ arising from the use of this software.
+
+ Permission is granted to anyone to use this software for any purpose,
+ including commercial applications, and to alter it and redistribute it
+ freely.
+*/
+
+extern SDL_bool SetupVideoToolboxOutput(SDL_Renderer *renderer);
+extern SDL_bool DisplayVideoToolboxFrame(SDL_Renderer *renderer, void *buffer, int srcX, int srcY, int srcW, int srcH, int dstX, int dstY, int dstW, int dstH );
+extern void CleanupVideoToolboxOutput();
diff --git a/test/testffmpeg_videotoolbox.m b/test/testffmpeg_videotoolbox.m
new file mode 100644
index 000000000000..e7d580cae0fb
--- /dev/null
+++ b/test/testffmpeg_videotoolbox.m
@@ -0,0 +1,147 @@
+/*
+ Copyright (C) 1997-2023 Sam Lantinga <slouken@libsdl.org>
+
+ This software is provided 'as-is', without any express or implied
+ warranty. In no event will the authors be held liable for any damages
+ arising from the use of this software.
+
+ Permission is granted to anyone to use this software for any purpose,
+ including commercial applications, and to alter it and redistribute it
+ freely.
+*/
+#include <SDL3/SDL.h>
+
+#include "testffmpeg_videotoolbox.h"
+
+#include <CoreVideo/CoreVideo.h>
+#include <Metal/Metal.h>
+#include <QuartzCore/CAMetalLayer.h>
+#include <simd/simd.h>
+
+
+// Metal BT.601 to RGB conversion shader
+static NSString *drawMetalShaderSource =
+@" using namespace metal;\n"
+"\n"
+" struct Vertex\n"
+" {\n"
+" float4 position [[position]];\n"
+" float2 texCoords;\n"
+" };\n"
+"\n"
+" constexpr sampler s(coord::normalized, address::clamp_to_edge, filter::linear);\n"
+"\n"
+" vertex Vertex draw_vs(constant Vertex *vertices [[ buffer(0) ]], uint vid [[ vertex_id ]])\n"
+" {\n"
+" return vertices[ vid ];\n"
+" }\n"
+"\n"
+" fragment float4 draw_ps_bt601(Vertex in [[ stage_in ]],\n"
+" texture2d<float> textureY [[ texture(0) ]],\n"
+" texture2d<float> textureUV [[ texture(1) ]])\n"
+" {\n"
+" float3 yuv = float3(textureY.sample(s, in.texCoords).r, textureUV.sample(s, in.texCoords).rg);\n"
+" float3 rgb;\n"
+" yuv += float3(-0.0627451017, -0.501960814, -0.501960814);\n"
+" rgb.r = dot(yuv, float3(1.1644, 0.000, 1.596));\n"
+" rgb.g = dot(yuv, float3(1.1644, -0.3918, -0.813));\n"
+" rgb.b = dot(yuv, float3(1.1644, 2.0172, 0.000));\n"
+" return float4(rgb, 1.0);\n"
+" }\n"
+;
+
+// keep this structure aligned with the proceeding drawMetalShaderSource's struct Vertex
+typedef struct Vertex
+{
+ vector_float4 position;
+ vector_float2 texCoord;
+} Vertex;
+
+static void SetVertex(Vertex *vertex, float x, float y, float s, float t)
+{
+ vertex->position[ 0 ] = x;
+ vertex->position[ 1 ] = y;
+ vertex->position[ 2 ] = 0.0f;
+ vertex->position[ 3 ] = 1.0f;
+ vertex->texCoord[ 0 ] = s;
+ vertex->texCoord[ 1 ] = t;
+}
+
+static CAMetalLayer *metal_layer;
+static id<MTLLibrary> library;
+static id<MTLRenderPipelineState> video_pipeline;
+
+SDL_bool SetupVideoToolboxOutput(SDL_Renderer *renderer)
+{ @autoreleasepool {
+ NSError *error;
+
+ // Create the metal view
+ metal_layer = (CAMetalLayer *)SDL_GetRenderMetalLayer(renderer);
+ if (!metal_layer) {
+ return SDL_FALSE;
+ }
+
+ // FIXME: Handle other colorspaces besides BT.601
+ library = [metal_layer.device newLibraryWithSource:drawMetalShaderSource options:nil error:&error];
+
+ MTLRenderPipelineDescriptor *videoPipelineDescriptor = [[MTLRenderPipelineDescriptor new] autorelease];
+ videoPipelineDescriptor.vertexFunction = [library newFunctionWithName:@"draw_vs"];
+ videoPipelineDescriptor.fragmentFunction = [library newFunctionWithName:@"draw_ps_bt601"];
+ videoPipelineDescriptor.colorAttachments[ 0 ].pixelFormat = metal_layer.pixelFormat;
+
+ video_pipeline = [metal_layer.device newRenderPipelineStateWithDescriptor:videoPipelineDescriptor error:nil];
+ if (!video_pipeline) {
+ SDL_SetError("Couldn't create video pipeline");
+ return SDL_FALSE;
+ }
+
+ return true;
+}}
+
+SDL_bool DisplayVideoToolboxFrame(SDL_Renderer *renderer, void *buffer, int srcX, int srcY, int srcW, int srcH, int dstX, int dstY, int dstW, int dstH )
+{ @autoreleasepool {
+ CVPixelBufferRef pPixelBuffer = (CVPixelBufferRef)buffer;
+ size_t nPixelBufferWidth = CVPixelBufferGetWidthOfPlane(pPixelBuffer, 0);
+ size_t nPixelBufferHeight = CVPixelBufferGetHeightOfPlane(pPixelBuffer, 0);
+ id<MTLTexture> videoFrameTextureY = nil;
+ id<MTLTexture> videoFrameTextureUV = nil;
+
+ IOSurfaceRef pSurface = CVPixelBufferGetIOSurface(pPixelBuffer);
+
+ MTLTextureDescriptor *textureDescriptorY = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatR8Unorm width:nPixelBufferWidth height:nPixelBufferHeight mipmapped:NO];
+ MTLTextureDescriptor *textureDescriptorUV = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRG8Unorm width:CVPixelBufferGetWidthOfPlane(pPixelBuffer, 1) height:CVPixelBufferGetHeightOfPlane(pPixelBuffer, 1) mipmapped:NO];
+
+ videoFrameTextureY = [[metal_layer.device newTextureWithDescriptor:textureDescriptorY iosurface:pSurface plane:0] autorelease];
+ videoFrameTextureUV = [[metal_layer.device newTextureWithDescriptor:textureDescriptorUV iosurface:pSurface plane:1] autorelease];
+
+ float flMinSrcX = ( srcX + 0.5f ) / nPixelBufferWidth;
+ float flMaxSrcX = ( srcX + srcW + 0.5f ) / nPixelBufferWidth;
+ float flMinSrcY = ( srcY + 0.5f ) / nPixelBufferHeight;
+ float flMaxSrcY = ( srcY + srcH + 0.5f ) / nPixelBufferHeight;
+
+ int nOutputWidth, nOutputHeight;
+ nOutputWidth = metal_layer.drawableSize.width;
+ nOutputHeight = metal_layer.drawableSize.height;
+ float flMinDstX = 2.0f * ( ( dstX + 0.5f ) / nOutputWidth ) - 1.0f;
+ float flMaxDstX = 2.0f * ( ( dstX + dstW + 0.5f ) / nOutputWidth ) - 1.0f;
+ float flMinDstY = 2.0f * ( ( nOutputHeight - dstY - 0.5f ) / nOutputHeight ) - 1.0f;
+ float flMaxDstY = 2.0f * ( ( nOutputHeight - ( dstY + dstH ) - 0.5f ) / nOutputHeight ) - 1.0f;
+
+ Vertex arrVerts[4];
+ SetVertex(&arrVerts[0], flMinDstX, flMaxDstY, flMinSrcX, flMaxSrcY);
+ SetVertex(&arrVerts[1], flMinDstX, flMinDstY, flMinSrcX, flMinSrcY);
+ SetVertex(&arrVerts[2], flMaxDstX, flMaxDstY, flMaxSrcX, flMaxSrcY);
+ SetVertex(&arrVerts[3], flMaxDstX, flMinDstY, flMaxSrcX, flMinSrcY);
+
+ id<MTLRenderCommandEncoder> renderEncoder = (id<MTLRenderCommandEncoder>)SDL_GetRenderMetalCommandEncoder(renderer);
+ [renderEncoder setRenderPipelineState:video_pipeline];
+ [renderEncoder setFragmentTexture:videoFrameTextureY atIndex:0];
+ [renderEncoder setFragmentTexture:videoFrameTextureUV atIndex:1];
+ [renderEncoder setVertexBytes:arrVerts length:sizeof(arrVerts) atIndex:0];
+ [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:SDL_arraysize(arrVerts)];
+ return SDL_TRUE;
+}}
+
+void CleanupVideoToolboxOutput()
+{
+}