From 332e7a050ed0d6ba6c9dc24850e91e2cf14a046f Mon Sep 17 00:00:00 2001
From: Sam Lantinga <[EMAIL REDACTED]>
Date: Fri, 17 Jul 2026 13:29:03 -0700
Subject: [PATCH] testffmpeg: added support for VA-API and DRM frames when
using Vulkan rendering
---
test/testffmpeg.c | 95 +++++++++-------
test/testffmpeg_vulkan.c | 237 ++++++++++++++++++++++++++++++++++++++-
test/testffmpeg_vulkan.h | 1 +
3 files changed, 294 insertions(+), 39 deletions(-)
diff --git a/test/testffmpeg.c b/test/testffmpeg.c
index 984df02f0141c..8ff1b93a49c84 100644
--- a/test/testffmpeg.c
+++ b/test/testffmpeg.c
@@ -81,6 +81,8 @@ static SDL_Window *window;
static SDL_Renderer *renderer;
static SDL_AudioStream *audio;
static SDL_Texture *video_texture;
+static AVCodecContext *audio_context;
+static AVCodecContext *video_context;
static Uint64 video_start;
static bool nodelay;
static bool software_only;
@@ -347,7 +349,8 @@ static bool SupportedPixelFormat(enum AVPixelFormat format)
return true;
}
#endif
- if (vulkan_context && format == AV_PIX_FMT_VULKAN) {
+ if (vulkan_context &&
+ (format == AV_PIX_FMT_VULKAN || format == AV_PIX_FMT_VAAPI || format == AV_PIX_FMT_DRM_PRIME)) {
return true;
}
}
@@ -497,15 +500,28 @@ static SDL_Colorspace GetFrameColorspace(AVFrame *frame)
SDL_Colorspace colorspace = SDL_COLORSPACE_SRGB;
if (frame && frame->colorspace != AVCOL_SPC_RGB) {
+ if (frame->colorspace != AVCOL_SPC_UNSPECIFIED ||
+ video_context->colorspace == AVCOL_SPC_UNSPECIFIED) {
#ifdef DEBUG_COLORSPACE
- SDL_Log("Frame colorspace: range: %d, primaries: %d, trc: %d, colorspace: %d, chroma_location: %d", frame->color_range, frame->color_primaries, frame->color_trc, frame->colorspace, frame->chroma_location);
+ SDL_Log("Frame colorspace: range: %d, primaries: %d, trc: %d, colorspace: %d, chroma_location: %d", frame->color_range, frame->color_primaries, frame->color_trc, frame->colorspace, frame->chroma_location);
#endif
- colorspace = SDL_DEFINE_COLORSPACE(SDL_COLOR_TYPE_YCBCR,
- frame->color_range,
- frame->color_primaries,
- frame->color_trc,
- frame->colorspace,
- frame->chroma_location);
+ colorspace = SDL_DEFINE_COLORSPACE(SDL_COLOR_TYPE_YCBCR,
+ frame->color_range,
+ frame->color_primaries,
+ frame->color_trc,
+ frame->colorspace,
+ frame->chroma_location);
+ } else if (video_context->colorspace != AVCOL_SPC_UNSPECIFIED) {
+#ifdef DEBUG_COLORSPACE
+ SDL_Log("Video colorspace: range: %d, primaries: %d, trc: %d, colorspace: %d, chroma_location: %d", video_context->color_range, video_context->color_primaries, video_context->color_trc, video_context->colorspace, video_context->chroma_sample_location);
+#endif
+ colorspace = SDL_DEFINE_COLORSPACE(SDL_COLOR_TYPE_YCBCR,
+ video_context->color_range,
+ video_context->color_primaries,
+ video_context->color_trc,
+ video_context->colorspace,
+ video_context->chroma_sample_location);
+ }
}
return colorspace;
}
@@ -654,9 +670,26 @@ static bool GetTextureForMemoryFrame(AVFrame *frame, SDL_Texture **texture)
return true;
}
+static bool GetTextureForVulkanFrame(AVFrame *frame, SDL_Texture **texture)
+{
+ SDL_PropertiesID props;
+
+ if (*texture) {
+ SDL_DestroyTexture(*texture);
+ }
+
+ props = CreateVideoTextureProperties(frame, SDL_PIXELFORMAT_UNKNOWN, SDL_TEXTUREACCESS_STATIC);
+ *texture = CreateVulkanVideoTexture(vulkan_context, frame, renderer, props);
+ SDL_DestroyProperties(props);
+ if (!*texture) {
+ return false;
+ }
+ return true;
+}
+
#ifdef HAVE_EGL
-static bool GetNV12TextureForDRMFrame(AVFrame *frame, SDL_Texture **texture)
+static bool GetEGLNV12TextureForDRMFrame(AVFrame *frame, SDL_Texture **texture)
{
AVHWFramesContext *frames = (AVHWFramesContext *)(frame->hw_frames_ctx ? frame->hw_frames_ctx->data : NULL);
const AVDRMFrameDescriptor *desc = (const AVDRMFrameDescriptor *)frame->data[0];
@@ -746,7 +779,7 @@ static bool GetNV12TextureForDRMFrame(AVFrame *frame, SDL_Texture **texture)
return true;
}
-static bool GetOESTextureForDRMFrame(AVFrame *frame, SDL_Texture **texture)
+static bool GetEGLOESTextureForDRMFrame(AVFrame *frame, SDL_Texture **texture)
{
AVHWFramesContext *frames = (AVHWFramesContext *)(frame->hw_frames_ctx ? frame->hw_frames_ctx->data : NULL);
const AVDRMFrameDescriptor *desc = (const AVDRMFrameDescriptor *)frame->data[0];
@@ -931,19 +964,24 @@ static bool GetOESTextureForDRMFrame(AVFrame *frame, SDL_Texture **texture)
static bool GetTextureForDRMFrame(AVFrame *frame, SDL_Texture **texture)
{
+ if (vulkan_context) {
+ return GetTextureForVulkanFrame(frame, texture);
+ } else {
#ifdef HAVE_EGL
- const AVDRMFrameDescriptor *desc = (const AVDRMFrameDescriptor *)frame->data[0];
+ const AVDRMFrameDescriptor *desc = (const AVDRMFrameDescriptor *)frame->data[0];
- if (desc->nb_layers == 2 &&
- desc->layers[0].format == DRM_FORMAT_R8 &&
- desc->layers[1].format == DRM_FORMAT_GR88) {
- return GetNV12TextureForDRMFrame(frame, texture);
- } else {
- return GetOESTextureForDRMFrame(frame, texture);
- }
+ if (desc->nb_layers == 2 &&
+ desc->layers[0].format == DRM_FORMAT_R8 &&
+ desc->layers[1].format == DRM_FORMAT_GR88) {
+ return GetEGLNV12TextureForDRMFrame(frame, texture);
+ } else {
+ return GetEGLOESTextureForDRMFrame(frame, texture);
+ }
#else
- return false;
+ SDL_Log("Creating EGL textures is not supported");
+ return false;
#endif
+ }
}
static bool GetTextureForVAAPIFrame(AVFrame *frame, SDL_Texture **texture)
@@ -1029,23 +1067,6 @@ static bool GetTextureForVideoToolboxFrame(AVFrame *frame, SDL_Texture **texture
#endif
}
-static bool GetTextureForVulkanFrame(AVFrame *frame, SDL_Texture **texture)
-{
- SDL_PropertiesID props;
-
- if (*texture) {
- SDL_DestroyTexture(*texture);
- }
-
- props = CreateVideoTextureProperties(frame, SDL_PIXELFORMAT_UNKNOWN, SDL_TEXTUREACCESS_STATIC);
- *texture = CreateVulkanVideoTexture(vulkan_context, frame, renderer, props);
- SDL_DestroyProperties(props);
- if (!*texture) {
- return false;
- }
- return true;
-}
-
static bool GetTextureForFrame(AVFrame *frame, SDL_Texture **texture)
{
switch (frame->format) {
@@ -1305,8 +1326,6 @@ int main(int argc, char *argv[])
const char *video_codec_name = NULL;
const AVCodec *audio_codec = NULL;
const AVCodec *video_codec = NULL;
- AVCodecContext *audio_context = NULL;
- AVCodecContext *video_context = NULL;
AVPacket *pkt = NULL;
AVFrame *frame = NULL;
double first_pts = -1.0;
diff --git a/test/testffmpeg_vulkan.c b/test/testffmpeg_vulkan.c
index f94a22df8b444..22504b138c07a 100644
--- a/test/testffmpeg_vulkan.c
+++ b/test/testffmpeg_vulkan.c
@@ -16,6 +16,24 @@
#ifdef FFMPEG_VULKAN_SUPPORT
+#ifndef SDL_PLATFORM_WINDOWS
+#define FFMPEG_DRMPRIME_SUPPORT
+
+#include <unistd.h>
+
+#define DRM_FORMAT_R8 SDL_FOURCC('R', '8', ' ', ' ')
+#define DRM_FORMAT_RG88 SDL_FOURCC('R', 'G', '8', '8')
+#define DRM_FORMAT_GR88 SDL_FOURCC('G', 'R', '8', '8')
+#define DRM_FORMAT_R16 SDL_FOURCC('R', '1', '6', ' ')
+#define DRM_FORMAT_RG1616 SDL_FOURCC('R', 'G', '3', '2')
+#define DRM_FORMAT_GR1616 SDL_FOURCC('G', 'R', '3', '2')
+#define DRM_FORMAT_NV12 SDL_FOURCC('N', 'V', '1', '2')
+#define DRM_FORMAT_P010 SDL_FOURCC('P', '0', '1', '0')
+#define DRM_FORMAT_YUYV SDL_FOURCC('Y', 'U', 'Y', 'V')
+#define DRM_FORMAT_UYVY SDL_FOURCC('U', 'Y', 'V', 'Y')
+#define DRM_FORMAT_ARGB8888 SDL_FOURCC('A', 'R', '2', '4')
+#endif
+
#define VULKAN_FUNCTIONS() \
VULKAN_GLOBAL_FUNCTION(vkCreateInstance) \
VULKAN_GLOBAL_FUNCTION(vkEnumerateInstanceExtensionProperties) \
@@ -27,13 +45,17 @@
VULKAN_INSTANCE_FUNCTION(vkEnumeratePhysicalDevices) \
VULKAN_INSTANCE_FUNCTION(vkGetDeviceProcAddr) \
VULKAN_INSTANCE_FUNCTION(vkGetPhysicalDeviceFeatures2) \
+ VULKAN_INSTANCE_FUNCTION(vkGetPhysicalDeviceMemoryProperties) \
VULKAN_INSTANCE_FUNCTION(vkGetPhysicalDeviceQueueFamilyProperties) \
VULKAN_INSTANCE_FUNCTION(vkGetPhysicalDeviceSurfaceSupportKHR) \
VULKAN_INSTANCE_FUNCTION(vkQueueWaitIdle) \
VULKAN_DEVICE_FUNCTION(vkAllocateCommandBuffers) \
+ VULKAN_DEVICE_FUNCTION(vkAllocateMemory) \
VULKAN_DEVICE_FUNCTION(vkBeginCommandBuffer) \
+ VULKAN_DEVICE_FUNCTION(vkBindImageMemory) \
VULKAN_DEVICE_FUNCTION(vkCmdPipelineBarrier2) \
VULKAN_DEVICE_FUNCTION(vkCreateCommandPool) \
+ VULKAN_DEVICE_FUNCTION(vkCreateImage) \
VULKAN_DEVICE_FUNCTION(vkCreateSemaphore) \
VULKAN_DEVICE_FUNCTION(vkDestroyCommandPool) \
VULKAN_DEVICE_FUNCTION(vkDestroyDevice) \
@@ -42,6 +64,7 @@
VULKAN_DEVICE_FUNCTION(vkEndCommandBuffer) \
VULKAN_DEVICE_FUNCTION(vkFreeCommandBuffers) \
VULKAN_DEVICE_FUNCTION(vkGetDeviceQueue) \
+ VULKAN_DEVICE_FUNCTION(vkGetImageMemoryRequirements) \
VULKAN_DEVICE_FUNCTION(vkQueueSubmit) \
\
VULKAN_INSTANCE_FUNCTION(vkGetPhysicalDeviceVideoFormatPropertiesKHR) \
@@ -864,7 +887,7 @@ int FinishVulkanFrameRendering(VulkanVideoContext *context, AVFrame *frame, SDL_
return 0;
}
-SDL_Texture *CreateVulkanVideoTexture(VulkanVideoContext *context, AVFrame *frame, SDL_Renderer *renderer, SDL_PropertiesID props)
+static SDL_Texture *CreateVulkanVideoTexturePixFmtVulkan(VulkanVideoContext *context, AVFrame *frame, SDL_Renderer *renderer, SDL_PropertiesID props)
{
AVHWFramesContext *frames = (AVHWFramesContext *)(frame->hw_frames_ctx->data);
AVVulkanFramesContext *vk = (AVVulkanFramesContext *)(frames->hwctx);
@@ -896,6 +919,218 @@ SDL_Texture *CreateVulkanVideoTexture(VulkanVideoContext *context, AVFrame *fram
return SDL_CreateTextureWithProperties(renderer, props);
}
+#ifdef FFMPEG_DRMPRIME_SUPPORT
+static bool FindMemoryIndex(VulkanVideoContext *context, uint32_t memoryTypeBits, uint32_t *memoryIndex)
+{
+ VkPhysicalDeviceMemoryProperties mem_properties;
+
+ context->vkGetPhysicalDeviceMemoryProperties(context->physicalDevice, &mem_properties);
+ for (uint32_t i = 0; i < mem_properties.memoryTypeCount; ++i) {
+ if (memoryTypeBits & (1 << i)) {
+ *memoryIndex = i;
+ return true;
+ }
+ }
+ return SDL_SetError("Couldn't find memory index for type %u", memoryTypeBits);
+}
+#endif /* FFMPEG_DRMPRIME_SUPPORT */
+
+static SDL_Texture *CreateVulkanVideoTexturePixFmtDRMPrime(VulkanVideoContext *context, AVFrame *frame, SDL_Renderer *renderer, SDL_PropertiesID props)
+{
+#ifdef FFMPEG_DRMPRIME_SUPPORT
+ const AVDRMFrameDescriptor *drm_desc = (const AVDRMFrameDescriptor *)frame->data[0];
+ VkFormat format = VK_FORMAT_UNDEFINED;
+ VkResult result;
+ SDL_Texture *texture;
+
+ if (drm_desc->nb_objects != 1) {
+ SDL_SetError("DRM frames with %d objects are not currently supported", drm_desc->nb_objects);
+ return NULL;
+ }
+
+ switch (drm_desc->layers[0].format) {
+ case DRM_FORMAT_R8:
+ if (drm_desc->nb_layers == 2) {
+ switch (drm_desc->layers[1].format) {
+ case DRM_FORMAT_GR88:
+ format = VK_FORMAT_G8_B8R8_2PLANE_420_UNORM;
+ SDL_SetNumberProperty(props, SDL_PROP_TEXTURE_CREATE_FORMAT_NUMBER, SDL_PIXELFORMAT_NV12);
+ break;
+ case DRM_FORMAT_RG88:
+ format = VK_FORMAT_G8_B8R8_2PLANE_420_UNORM;
+ SDL_SetNumberProperty(props, SDL_PROP_TEXTURE_CREATE_FORMAT_NUMBER, SDL_PIXELFORMAT_NV21);
+ break;
+ default:
+ break;
+ }
+ }
+ break;
+ case DRM_FORMAT_R16:
+ if (drm_desc->nb_layers == 2) {
+ switch (drm_desc->layers[1].format) {
+ case DRM_FORMAT_GR1616:
+ format = VK_FORMAT_G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16;
+ SDL_SetNumberProperty(props, SDL_PROP_TEXTURE_CREATE_FORMAT_NUMBER, SDL_PIXELFORMAT_P010);
+ break;
+ default:
+ break;
+ }
+ }
+ break;
+ case DRM_FORMAT_NV12:
+ format = VK_FORMAT_G8_B8R8_2PLANE_420_UNORM;
+ SDL_SetNumberProperty(props, SDL_PROP_TEXTURE_CREATE_FORMAT_NUMBER, SDL_PIXELFORMAT_NV12);
+ break;
+ case DRM_FORMAT_P010:
+ format = VK_FORMAT_G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16;
+ SDL_SetNumberProperty(props, SDL_PROP_TEXTURE_CREATE_FORMAT_NUMBER, SDL_PIXELFORMAT_P010);
+ break;
+ case DRM_FORMAT_YUYV:
+ format = VK_FORMAT_G8B8G8R8_422_UNORM;
+ SDL_SetNumberProperty(props, SDL_PROP_TEXTURE_CREATE_FORMAT_NUMBER, SDL_PIXELFORMAT_YUY2);
+ break;
+ case DRM_FORMAT_UYVY:
+ format = VK_FORMAT_B8G8R8G8_422_UNORM;
+ SDL_SetNumberProperty(props, SDL_PROP_TEXTURE_CREATE_FORMAT_NUMBER, SDL_PIXELFORMAT_UYVY);
+ break;
+ case DRM_FORMAT_ARGB8888:
+ format = VK_FORMAT_B8G8R8A8_UNORM;
+ SDL_SetNumberProperty(props, SDL_PROP_TEXTURE_CREATE_FORMAT_NUMBER, SDL_PIXELFORMAT_BGRA32);
+ break;
+ default:
+ break;
+ }
+ if (format == VK_FORMAT_UNDEFINED) {
+ SDL_SetError("Unsupported DRM format %d", drm_desc->layers[0].format);
+ return NULL;
+ }
+
+ int dma_buf_fds[AV_DRM_MAX_PLANES];
+ for (int i = 0; i < drm_desc->nb_objects; i++) {
+ // Duplicate the descriptor if your Vulkan driver or framework closes it automatically
+ dma_buf_fds[i] = dup(drm_desc->objects[i].fd);
+ if (dma_buf_fds[i] < 0) {
+ while (--i >= 0) {
+ close(dma_buf_fds[i]);
+ }
+ SDL_SetError("Couldn't duplicate file descriptor");
+ return NULL;
+ }
+ }
+
+ // Map your descriptor planes to Vulkan plane layouts
+ uint32_t planes = 0;
+ VkSubresourceLayout plane_layouts[AV_DRM_MAX_PLANES];
+ SDL_zeroa(plane_layouts);
+ for (int i = 0; i < drm_desc->nb_layers; ++i) {
+ for (int j = 0; j < drm_desc->layers[i].nb_planes; ++j) {
+ const AVDRMPlaneDescriptor *plane = &drm_desc->layers[i].planes[j];
+
+ plane_layouts[planes].offset = plane->offset;
+ plane_layouts[planes].rowPitch = plane->pitch;
+ ++planes;
+ }
+ }
+
+ VkImageDrmFormatModifierExplicitCreateInfoEXT modifier_info;
+ SDL_zero(modifier_info);
+ modifier_info.sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_EXPLICIT_CREATE_INFO_EXT;
+ modifier_info.drmFormatModifier = drm_desc->objects[0].format_modifier;
+ modifier_info.drmFormatModifierPlaneCount = planes;
+ modifier_info.pPlaneLayouts = plane_layouts;
+
+ VkExternalMemoryImageCreateInfo external_info;
+ SDL_zero(external_info);
+ external_info.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO;
+ external_info.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT;
+ external_info.pNext = &modifier_info;
+
+ VkImageCreateInfo image_info;
+ SDL_zero(image_info);
+ image_info.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
+ image_info.imageType = VK_IMAGE_TYPE_2D;
+ image_info.format = format;
+ image_info.extent.width = frame->width;
+ image_info.extent.height = frame->height;
+ image_info.extent.depth = 1;
+ image_info.mipLevels = 1;
+ image_info.arrayLayers = 1;
+ image_info.samples = VK_SAMPLE_COUNT_1_BIT;
+ image_info.tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT;
+ image_info.usage = VK_IMAGE_USAGE_SAMPLED_BIT;
+ image_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
+ image_info.pNext = &external_info;
+
+ VkImage image;
+ result = context->vkCreateImage(context->device, &image_info, NULL, &image);
+ if (result != VK_SUCCESS) {
+ SDL_SetError("vkCreateImage(): %s", getVulkanResultString(result));
+ goto error;
+ }
+ SDL_SetNumberProperty(props, SDL_PROP_TEXTURE_CREATE_VULKAN_TEXTURE_NUMBER, (Sint64)image);
+
+ // Calculate total memory size from plane requirements
+ VkMemoryRequirements mem_reqs;
+ context->vkGetImageMemoryRequirements(context->device, image, &mem_reqs);
+
+ VkImportMemoryFdInfoKHR import_fd_info;
+ SDL_zero(import_fd_info);
+ import_fd_info.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR;
+ import_fd_info.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT;
+ import_fd_info.fd = dma_buf_fds[0];
+
+ uint32_t memoryTypeIndex = 0;
+ if (!FindMemoryIndex(context, mem_reqs.memoryTypeBits, &memoryTypeIndex)) {
+ goto error;
+ }
+ VkMemoryAllocateInfo alloc_info;
+ SDL_zero(alloc_info);
+ alloc_info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
+ alloc_info.allocationSize = mem_reqs.size;
+ alloc_info.memoryTypeIndex = memoryTypeIndex;
+ alloc_info.pNext = &import_fd_info;
+
+ VkDeviceMemory image_memory;
+ result = context->vkAllocateMemory(context->device, &alloc_info, NULL, &image_memory);
+ if (result != VK_SUCCESS) {
+ SDL_SetError("vkAllocateMemory(): %s", getVulkanResultString(result));
+ goto error;
+ }
+ result = context->vkBindImageMemory(context->device, image, image_memory, 0);
+ if (result != VK_SUCCESS) {
+ SDL_SetError("vkBindImageMemory(): %s", getVulkanResultString(result));
+ goto error;
+ }
+
+ texture = SDL_CreateTextureWithProperties(renderer, props);
+ if (!texture) {
+ goto error;
+ }
+ return texture;
+
+error:
+ for (int i = 0; i < drm_desc->nb_objects; i++) {
+ close(dma_buf_fds[i]);
+ }
+ return NULL;
+#else
+ SDL_SetError("DRM prime frames not supported");
+ return NULL;
+#endif /* FFMPEG_DRMPRIME_SUPPORT */
+}
+
+SDL_Texture *CreateVulkanVideoTexture(VulkanVideoContext *context, AVFrame *frame, SDL_Renderer *renderer, SDL_PropertiesID props)
+{
+ if (frame->format == AV_PIX_FMT_VULKAN) {
+ return CreateVulkanVideoTexturePixFmtVulkan(context, frame, renderer, props);
+ } else if (frame->format == AV_PIX_FMT_DRM_PRIME) {
+ return CreateVulkanVideoTexturePixFmtDRMPrime(context, frame, renderer, props);
+ } else {
+ SDL_SetError("Unknown hardware frame format");
+ return NULL;
+ }
+}
+
void DestroyVulkanVideoContext(VulkanVideoContext *context)
{
if (context) {
diff --git a/test/testffmpeg_vulkan.h b/test/testffmpeg_vulkan.h
index e61f36cd2512c..7b877aa6a13cd 100644
--- a/test/testffmpeg_vulkan.h
+++ b/test/testffmpeg_vulkan.h
@@ -11,6 +11,7 @@
*/
#include <libavutil/hwcontext.h>
+#include <libavutil/hwcontext_drm.h>
#include <libavutil/hwcontext_vulkan.h>