SDL GStreamer interop via Vulkan

SDL Development
1

I’ve attempted to implement a simple demo, however, it only draw the all-time classic black image onto the window.

main.c:

#define _GNU_SOURCE
#include <assert.h>
#include <stdio.h>
#include <time.h>
#include <pthread.h>

#include <SDL3/SDL.h>
#include <gst/gst.h>
#include <gst/app/app.h>
#include <gst/vulkan/vulkan.h>

typedef struct {
    GMutex mutex;

    VkInstance instance;
    VkPhysicalDevice physical_device;
    VkDevice device;

    GMainLoop *loop;

    GQueue queue;

    GstPipeline *pipeline;

    SDL_Window *window;
    SDL_Renderer *renderer;

    SDL_Texture *texture;
    GstVulkanImageMemory *vk_memory;
} AppData;

static void end_stream_cb(GstBus *bus, GstMessage *msg, AppData *appdata) {
    switch (GST_MESSAGE_TYPE (msg)) {
        case GST_MESSAGE_EOS:
            SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "End of stream");
            g_main_loop_quit(appdata->loop);
            break;
        default:
            break;
    }
}

static SDL_Renderer *create_renderer(AppData *appdata) {
    SDL_PropertiesID props = SDL_CreateProperties();
    SDL_SetPointerProperty(props, SDL_PROP_RENDERER_CREATE_WINDOW_POINTER, appdata->window);
    SDL_SetPointerProperty(props, SDL_PROP_RENDERER_CREATE_VULKAN_INSTANCE_POINTER, appdata->instance);
    SDL_SetPointerProperty(props, SDL_PROP_RENDERER_CREATE_VULKAN_PHYSICAL_DEVICE_POINTER, appdata->physical_device);
    SDL_Renderer *renderer = SDL_CreateRendererWithProperties(props);
    SDL_DestroyProperties(props);
    if (renderer == NULL) {
        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Failed to create renderer: %s", SDL_GetError());
    } else {
        SDL_ShowWindow(appdata->window);
    }
    return renderer;
}

static SDL_Texture *create_texture(AppData *appdata) {
    SDL_PropertiesID props = SDL_CreateProperties();
    SDL_SetNumberProperty(props, SDL_PROP_TEXTURE_CREATE_WIDTH_NUMBER, gst_vulkan_image_memory_get_width(appdata->vk_memory));
    SDL_SetNumberProperty(props, SDL_PROP_TEXTURE_CREATE_HEIGHT_NUMBER, gst_vulkan_image_memory_get_height(appdata->vk_memory));
    SDL_SetNumberProperty(props, SDL_PROP_TEXTURE_CREATE_FORMAT_NUMBER, SDL_PIXELFORMAT_ABGR8888);
    SDL_SetNumberProperty(props, SDL_PROP_TEXTURE_CREATE_VULKAN_TEXTURE_NUMBER, (Sint64) appdata->vk_memory->image);
    SDL_Texture *texture = SDL_CreateTextureWithProperties(appdata->renderer, props);
    SDL_DestroyProperties(props);
    if (texture == NULL) {
        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Failed to create texture: %s", SDL_GetError());
    }
    return texture;
}

static const char *vk_phys_dev_types[] = {
    [VK_PHYSICAL_DEVICE_TYPE_OTHER] = "WTF",
    [VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU] = "iGPU",
    [VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU] = "dGPU",
    [VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU] = "vGPU",
    [VK_PHYSICAL_DEVICE_TYPE_CPU] = "CPU",
};

static void sync_bus_call(GstBus *bus, GstMessage *msg, gpointer data) {
    AppData *appdata = data;
    switch (GST_MESSAGE_TYPE (msg)) {
        case GST_MESSAGE_NEED_CONTEXT:
            const gchar *context_type = NULL;
            gst_message_parse_context_type(msg, &context_type);

            SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Requested context type %s", context_type);
            break;
        case GST_MESSAGE_HAVE_CONTEXT:
            GstContext *context;
            gst_message_parse_have_context(msg, &context);
            context_type = gst_context_get_context_type(context);
            SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Have context type %s", context_type);
            if (g_strcmp0(context_type, GST_VULKAN_INSTANCE_CONTEXT_TYPE_STR) == 0) {
                GstVulkanInstance *instance = NULL;
                gst_context_get_vulkan_instance(context, &instance);
                SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "[GST] VkInstance: %p", instance->instance);
                SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "[GST] VkPhysicalDevices: %p", instance->physical_devices);
                SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "[GST] n_physical_devices: %u", instance->n_physical_devices);

                assert(instance->n_physical_devices > 0);
                VkPhysicalDeviceProperties props = {};
                vkGetPhysicalDeviceProperties(instance->physical_devices[0], &props);
                assert(props.deviceType != VK_PHYSICAL_DEVICE_TYPE_CPU);

                for (size_t i = 0; i < instance->n_physical_devices; i++) {
                    VkPhysicalDeviceProperties props = {};
                    vkGetPhysicalDeviceProperties(instance->physical_devices[i], &props);
                    SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "[GST] VkPhysicalDevice[%zu]: name %s", i, props.deviceName);
                    SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "[GST] VkPhysicalDevice[%zu]: type %s", i, vk_phys_dev_types[props.deviceType]);
                }

                appdata->instance = instance->instance;
                appdata->physical_device = instance->physical_devices[0];
                appdata->renderer = create_renderer(appdata);
            }
            gst_context_unref (context);
            break;
        default:
            break;
    }
}

#define NANOS_PER_SEC (1000*1000*1000)
uint64_t nanos_since_unspecified_epoch(void)
{
    struct timespec ts;
    clock_gettime(CLOCK_MONOTONIC, &ts);

    return NANOS_PER_SEC * ts.tv_sec + ts.tv_nsec;
}

#define check(what, msg) \
    if (!(what)) { \
        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, msg ": %s", SDL_GetError()); \
        g_main_loop_quit(appdata->loop); \
        return G_SOURCE_REMOVE; \
    }

static gboolean handle_input_and_draw_unlocked(AppData *appdata) {
    char THREAD_NAME[128] = {}; pthread_getname_np(pthread_self(), THREAD_NAME, sizeof(THREAD_NAME));

    if (appdata->renderer == NULL) {
        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "GST VK is not initialized yet...");
        return G_SOURCE_CONTINUE;
    }
    SDL_Event event;
    while (SDL_PollEvent(&event)) {
        if (event.type == SDL_EVENT_QUIT) {
            SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "SDL_EVENT_QUIT");
            // gst_element_send_event(GST_ELEMENT(appdata->pipeline), gst_event_new_eos()); // this hangs
            g_main_loop_quit(appdata->loop);
            return G_SOURCE_REMOVE;
        }
    }

    GstVulkanImageMemory *next_memory = g_queue_pop_head(&appdata->queue);
    if (next_memory != NULL) {
        if (appdata->texture != NULL) {
            SDL_DestroyTexture(appdata->texture);
            appdata->texture = NULL;
        }
        if (appdata->vk_memory != NULL) {
            gst_memory_unref(GST_MEMORY_CAST(appdata->vk_memory));
            appdata->vk_memory = NULL;
        }
        appdata->vk_memory = next_memory;
        appdata->texture = create_texture(appdata);
    } else if (appdata->texture == NULL) {
        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "[%s] Empty queue and no texture yet", THREAD_NAME);
    }

    if (appdata->texture == NULL) {
        check(SDL_SetRenderDrawColor(appdata->renderer, 0xFF, 0x00, 0x00, 0xFF), "Failed to set color");
        check(SDL_RenderClear(appdata->renderer), "Failed to clear with color");
    } else {
        check(SDL_RenderTexture(appdata->renderer, appdata->texture, NULL, NULL), "Failed to render texture");
    }
    check(SDL_RenderPresent(appdata->renderer), "Failed to present");

    static uint64_t prev = 0;
    if (prev != 0) {
        uint64_t curr = nanos_since_unspecified_epoch();
        double diff = (curr - prev) / (double) NANOS_PER_SEC;
        prev = curr;
        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "[%s] Time between draw: %f s / %f FPS", THREAD_NAME, diff, 1 / diff);
    } else {
        prev = nanos_since_unspecified_epoch();
    }
    return G_SOURCE_CONTINUE;
}
#undef check

static gboolean handle_input_and_draw(gpointer data) {
    AppData *appdata = data;
    g_mutex_lock(&appdata->mutex);
    gboolean result = handle_input_and_draw_unlocked(appdata);
    g_mutex_unlock(&appdata->mutex);

    return result;
}

GstFlowReturn new_sample_cb(GstElement *object, gpointer data) {
    AppData *appdata = (AppData *) data;

    GstSample *sample = gst_app_sink_pull_sample(GST_APP_SINK(object));
    if (sample == NULL) {
        return GST_FLOW_EOS;
    }

    GstBuffer *buffer = gst_sample_get_buffer(sample);
    gchar *caps = gst_caps_to_string(gst_sample_get_caps(sample));
    g_free(caps);

    // TODO: get GstVulkanImageMemory->device and maybe initialize window here
    // or how else would it choose the proper GPU?
    // Or better yet, make GStreamer use the SDL VkInstance... There is no such API yet though.
    g_mutex_lock(&appdata->mutex);

    assert(gst_buffer_n_memory(buffer) == 1);
    GstMemory *memory = gst_buffer_peek_memory(buffer, 0);
    assert(gst_is_vulkan_image_memory(memory));

    g_queue_push_tail(&appdata->queue, gst_memory_ref(memory));

    // handle_input_and_draw_unlocked(appdata); // must not call this here as it needs to be in the main thread
    g_mutex_unlock(&appdata->mutex);
    gst_sample_unref(sample);
    return GST_FLOW_OK;
}

#define check(what, msg) \
    if (!(what)) { \
        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, msg ": %s", SDL_GetError()); \
        return 1; \
    }

int main(int argc, char **argv) {
    check(SDL_Init(SDL_INIT_VIDEO), "Failed to initialize SDL");

    SDL_Window *window = SDL_CreateWindow("SDL GStreamer Vulkan Demo", 1280, 800, SDL_WINDOW_RESIZABLE | SDL_WINDOW_VULKAN | SDL_WINDOW_HIDDEN);
    check(window != NULL, "Failed to create SDL Vulkan window");

    gst_init (&argc, &argv);
    GMainLoop *loop = g_main_loop_new(NULL, FALSE);

    GError *error = NULL;
    GstPipeline *pipeline = GST_PIPELINE(gst_parse_launch("videotestsrc ! video/x-raw,format=NV12,width=1280,height=800,framerate=60/1 ! vulkanupload ! vulkancolorconvert ! video/x-raw(memory:VulkanImage),format=RGBA ! appsink name=vksink emit-signals=true", &error));
    assert(pipeline != NULL && error == NULL);

    AppData appdata = {
        .loop = loop,
        .pipeline = pipeline,
        .window = window,
        .queue = G_QUEUE_INIT,
    };

    g_mutex_init(&appdata.mutex);

    GstAppSink *vksink = GST_APP_SINK(gst_bin_get_by_name(GST_BIN(pipeline), "vksink"));
    g_signal_connect(vksink, "new-sample", G_CALLBACK(new_sample_cb), &appdata);

    GstBus *bus = gst_pipeline_get_bus(pipeline);
    gst_bus_add_signal_watch(bus);
    g_signal_connect(bus, "message::eos", G_CALLBACK(end_stream_cb), &appdata);
    gst_bus_enable_sync_message_emission(bus);
    g_signal_connect(bus, "sync-message", G_CALLBACK(sync_bus_call), &appdata);
    assert(pipeline != NULL);

    gst_element_set_state(GST_ELEMENT(pipeline), GST_STATE_PAUSED);
    gst_element_set_state(GST_ELEMENT(pipeline), GST_STATE_PLAYING);

    g_timeout_add(10, handle_input_and_draw, &appdata);

    g_main_loop_run(loop);
    gst_element_set_state(GST_ELEMENT(pipeline), GST_STATE_NULL);
    gst_object_unref(pipeline);

    SDL_DestroyRenderer(appdata.renderer);
    SDL_DestroyWindow(window);

    SDL_Quit();
}

Makefile:

CC?=gcc
AR?=ar
CXX?=g++

INCLUDES=
DEFINES=
LIBS=

PKGCONF=gstreamer-1.0 \
        gstreamer-app-1.0 \
        gstreamer-vulkan-1.0 \
        vulkan \
        sdl3

CFLAGS=-ggdb -fPIC -O3 $(addprefix -I,$(INCLUDES)) $(addprefix -D,$(DEFINES)) $(shell pkg-config --cflags $(PKGCONF))
LDFLAGS=$(addprefix -l,$(LIBS)) $(shell pkg-config --libs $(PKGCONF))

all: build/main

MAIN_EXE_OBJS=$(addprefix build/,main.o)

build/main: $(MAIN_EXE_OBJS) | build
	$(CC) $(LDFLAGS) -o $@ $^

build/%.o: %.c | build
	$(CC) $(CFLAGS) $^ -c -o $@

build:
	mkdir -pv $@

Any ideas what could be wrong?

2

Yep, you can make sdl + gstreamer play nice with vulkan. easiest path: use appsink to pull RGBA frames and upload them into a VkImage you present via SDL’s Vulkan surface (SDL_Vulkan_CreateSurface). It’s a copy per frame, but super reliable.

If you want near zero-copy, go the DMABUF/extern-memory route: gstreamer vulkan (or VAAPI) → dmabuf → import with VK_EXT_external_memory_dma_buf/VK_KHR_external_memory and sample that image in your render pass, then present to the SDL swapchain. More setup, but fast.

Total shortcut: if you don’t need vulkan effects, decode to RGBA and push into an SDL_Texture with the regular renderer—simplest to ship.

3

Yeah, of course I can copy between GPU and CPU memory back and forth, that’s not the point. This needs to work as fast as possible on Linux, Windows and MacOS. And the only common denominator between them is Vulkan (Linux and Windows seem to both support Vulkan video decoding and MoltenVK seems to support importing HW decoded video frames for the MacOS). I was hoping that SDL_PROP_TEXTURE_CREATE_VULKAN_TEXTURE_NUMBER would just work as I expect it to, but of course it doesn’t…

Also, at some point I’d have to integrate an UI overlay on top of the video. In the current setup, this seems to be as easy as to use Dear ImGUI and call ImGui_ImplSDLRenderer3_RenderDrawData between SDL_RenderTexture and SDL_RenderPresent.

Is there any way to debug what exactly lead to my code not working?

4

This is quite stupid, but it turns out setting SDL_PROP_RENDERER_CREATE_VULKAN_*_POINTER is not enough to force Vulkan rendering. And even if I do so by setting the renderer name, it fails:

ERROR: Failed to create renderer: vulkan not available

With SDL_RENDER_VULKAN_DEBUG=1 I’m also seeing this:

ERROR: Vulkan_CreateSurface() failed
5

SDL needs VK_KHR_xlib_surface, apparently. Who would’ve guessed…

Now I see at least something on the screen:

image
image1280×828 16 KB

Is this some common mistake with Vulkan? Format seems to be correct, it just crashed with assert until I fixed it from one permutation of RGBA to another one, so it’s definitely something else.

6

The issue was that VkImage is tied to a specific VkDevice, but there is no way to get one from GStreamer before actually getting first frame.