SDL: wayland: Allocate the cursor shared memory buffer entirely in memory, if possible.

From 9bdb992925e853d1a9dd85e70ee8a40561e8373d Mon Sep 17 00:00:00 2001
From: Frank Praznik <[EMAIL REDACTED]>
Date: Tue, 12 Mar 2024 17:39:32 -0400
Subject: [PATCH] wayland: Allocate the cursor shared memory buffer entirely in
 memory, if possible.

Use memfd_create() to allocate the temporary SHM backing file in memory, and set the size with posix_fallocate(), which will return an error on insufficient space vs ftruncate(), which will silently succeed and allow a SIGBUS error to occur if the unbacked memory is accessed.

Additionally, make the legacy path more robust by unlinking the temp file, so it won't persist after close, and unmapping the shared memory buffer.
---
 CMakeLists.txt                                |  2 +
 include/build_config/SDL_build_config.h.cmake |  2 +
 src/video/wayland/SDL_waylandmouse.c          | 85 ++++++++++++++-----
 3 files changed, 70 insertions(+), 19 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9e5e4bcf094bf..3c09a00db511c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1107,6 +1107,8 @@ if(SDL_LIBC)
     check_symbol_exists(getauxval "sys/auxv.h" HAVE_GETAUXVAL)
     check_symbol_exists(elf_aux_info "sys/auxv.h" HAVE_ELF_AUX_INFO)
     check_symbol_exists(poll "poll.h" HAVE_POLL)
+    check_symbol_exists(memfd_create "sys/mman.h" HAVE_MEMFD_CREATE)
+    check_symbol_exists(posix_fallocate "fcntl.h" HAVE_POSIX_FALLOCATE)
 
     if(SDL_SYSTEM_ICONV)
       check_c_source_compiles("
diff --git a/include/build_config/SDL_build_config.h.cmake b/include/build_config/SDL_build_config.h.cmake
index e6f5d91c31735..606dbbe7d21dc 100644
--- a/include/build_config/SDL_build_config.h.cmake
+++ b/include/build_config/SDL_build_config.h.cmake
@@ -186,6 +186,8 @@
 #cmakedefine HAVE_FOPEN64 1
 #cmakedefine HAVE_FSEEKO 1
 #cmakedefine HAVE_FSEEKO64 1
+#cmakedefine HAVE_MEMFD_CREATE 1
+#cmakedefine HAVE_POSIX_FALLOCATE 1
 #cmakedefine HAVE_SIGACTION 1
 #cmakedefine HAVE_SA_SIGACTION 1
 #cmakedefine HAVE_SETJMP 1
diff --git a/src/video/wayland/SDL_waylandmouse.c b/src/video/wayland/SDL_waylandmouse.c
index a8416c5e0cf1c..13b6e807c3f69 100644
--- a/src/video/wayland/SDL_waylandmouse.c
+++ b/src/video/wayland/SDL_waylandmouse.c
@@ -23,11 +23,12 @@
 
 #ifdef SDL_VIDEO_DRIVER_WAYLAND
 
-#include <sys/types.h>
 #include <sys/mman.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <limits.h>
+#include <signal.h>
+#include <errno.h>
 
 #include "../SDL_sysvideo.h"
 #include "../SDL_video_c.h"
@@ -59,6 +60,7 @@ typedef struct
      */
     SDL_SystemCursor system_cursor;
     void *shm_data;
+    size_t shm_data_size;
 } Wayland_CursorData;
 
 static int dbus_cursor_size;
@@ -346,27 +348,72 @@ static SDL_bool wayland_get_system_cursor(SDL_VideoData *vdata, Wayland_CursorDa
     return SDL_TRUE;
 }
 
-static int wayland_create_tmp_file(off_t size)
+static int set_tmp_file_size(int fd, off_t size)
 {
-    static const char template[] = "/sdl-shared-XXXXXX";
-    char *xdg_path;
-    char tmp_path[PATH_MAX];
-    int fd;
+#ifdef HAVE_POSIX_FALLOCATE
+    sigset_t set, old_set;
+    int ret;
 
-    xdg_path = SDL_getenv("XDG_RUNTIME_DIR");
-    if (!xdg_path) {
-        return -1;
-    }
+    /* SIGALRM can potentially block a large posix_fallocate() operation
+     * from succeeding, so block it.
+     */
+    sigemptyset(&set);
+    sigaddset(&set, SIGALRM);
+    sigprocmask(SIG_BLOCK, &set, &old_set);
+
+    do {
+        ret = posix_fallocate(fd, 0, size);
+    } while (ret == EINTR);
 
-    SDL_strlcpy(tmp_path, xdg_path, PATH_MAX);
-    SDL_strlcat(tmp_path, template, PATH_MAX);
+    sigprocmask(SIG_SETMASK, &old_set, NULL);
 
-    fd = mkostemp(tmp_path, O_CLOEXEC);
-    if (fd < 0) {
+    if (ret == 0) {
+        return 0;
+    }
+    else if (ret != EINVAL && errno != EOPNOTSUPP) {
         return -1;
     }
+#endif
 
     if (ftruncate(fd, size) < 0) {
+        return -1;
+    }
+    return 0;
+}
+
+static int wayland_create_tmp_file(off_t size)
+{
+    int fd;
+
+#ifdef HAVE_MEMFD_CREATE
+    fd = memfd_create("SDL", MFD_CLOEXEC | MFD_ALLOW_SEALING);
+    if (fd >= 0) {
+        fcntl(fd, F_ADD_SEALS, F_SEAL_SHRINK | F_SEAL_SEAL);
+    } else
+#endif
+    {
+        static const char template[] = "/sdl-shared-XXXXXX";
+        char *xdg_path;
+        char tmp_path[PATH_MAX];
+
+        xdg_path = SDL_getenv("XDG_RUNTIME_DIR");
+        if (!xdg_path) {
+            return -1;
+        }
+
+        SDL_strlcpy(tmp_path, xdg_path, PATH_MAX);
+        SDL_strlcat(tmp_path, template, PATH_MAX);
+
+        fd = mkostemp(tmp_path, O_CLOEXEC);
+        if (fd < 0) {
+            return -1;
+        }
+
+        /* Need to manually unlink the temp files, or they can persist after close and fill up the temp storage. */
+        unlink(tmp_path);
+    }
+
+    if (set_tmp_file_size(fd, size) < 0) {
         close(fd);
         return -1;
     }
@@ -392,17 +439,17 @@ static int create_buffer_from_shm(Wayland_CursorData *d,
     struct wl_shm_pool *shm_pool;
 
     int stride = width * 4;
-    int size = stride * height;
+    d->shm_data_size = stride * height;
 
     int shm_fd;
 
-    shm_fd = wayland_create_tmp_file(size);
+    shm_fd = wayland_create_tmp_file(d->shm_data_size);
     if (shm_fd < 0) {
         return SDL_SetError("Creating mouse cursor buffer failed.");
     }
 
     d->shm_data = mmap(NULL,
-                       size,
+                       d->shm_data_size,
                        PROT_READ | PROT_WRITE,
                        MAP_SHARED,
                        shm_fd,
@@ -415,7 +462,7 @@ static int create_buffer_from_shm(Wayland_CursorData *d,
 
     SDL_assert(d->shm_data != NULL);
 
-    shm_pool = wl_shm_create_pool(data->shm, shm_fd, size);
+    shm_pool = wl_shm_create_pool(data->shm, shm_fd, d->shm_data_size);
     d->buffer = wl_shm_pool_create_buffer(shm_pool,
                                           0,
                                           width,
@@ -506,6 +553,7 @@ static void Wayland_FreeCursorData(Wayland_CursorData *d)
     if (d->buffer) {
         if (d->shm_data) {
             wl_buffer_destroy(d->buffer);
+            munmap(d->shm_data, d->shm_data_size);
         }
         d->buffer = NULL;
     }
@@ -529,7 +577,6 @@ static void Wayland_FreeCursor(SDL_Cursor *cursor)
 
     Wayland_FreeCursorData((Wayland_CursorData *)cursor->driverdata);
 
-    /* Not sure what's meant to happen to shm_data */
     SDL_free(cursor->driverdata);
     SDL_free(cursor);
 }