SDL: wayland: Allocate the cursor shared memory buffer entirely in memory, if possible. (d8490)

From d84903692724d5d4a3d1e2c533e9599045b4661d Mon Sep 17 00:00:00 2001
From: Frank Praznik <[EMAIL REDACTED]>
Date: Tue, 12 Mar 2024 17:39:32 -0400
Subject: [PATCH] wayland: Allocate the cursor shared memory buffer entirely in
 memory, if possible.

Use memfd_create() to allocate the temporary SHM backing file in memory, and set the size with posix_fallocate(), which will return an error on insufficient space vs ftruncate(), which will silently succeed and allow a SIGBUS error to occur if the unbacked memory is accessed.

Additionally, make the legacy path more robust by unlinking the temp file, so it won't persist after close, and unmapping the shared memory buffer.

(cherry picked from commit 9bdb992925e853d1a9dd85e70ee8a40561e8373d)
(cherry picked from commit d3c89bb4794b40fd0a74caa66ba2557971eab225)
---
 CMakeLists.txt                       |  2 +
 include/SDL_config.h.cmake           |  2 +
 src/video/wayland/SDL_waylandmouse.c | 85 +++++++++++++++++++++-------
 3 files changed, 70 insertions(+), 19 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0db9cba62b861..540992055f64c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1107,6 +1107,8 @@ if(SDL_LIBC)
     check_symbol_exists(getauxval "sys/auxv.h" HAVE_GETAUXVAL)
     check_symbol_exists(elf_aux_info "sys/auxv.h" HAVE_ELF_AUX_INFO)
     check_symbol_exists(poll "poll.h" HAVE_POLL)
+    check_symbol_exists(memfd_create "sys/mman.h" HAVE_MEMFD_CREATE)
+    check_symbol_exists(posix_fallocate "fcntl.h" HAVE_POSIX_FALLOCATE)
 
     check_library_exists(m pow "" HAVE_LIBM)
     if(HAVE_LIBM)
diff --git a/include/SDL_config.h.cmake b/include/SDL_config.h.cmake
index e1da0c6d4d5c6..ffcafd895ce8a 100644
--- a/include/SDL_config.h.cmake
+++ b/include/SDL_config.h.cmake
@@ -190,6 +190,8 @@
 #cmakedefine HAVE_FOPEN64 1
 #cmakedefine HAVE_FSEEKO 1
 #cmakedefine HAVE_FSEEKO64 1
+#cmakedefine HAVE_MEMFD_CREATE 1
+#cmakedefine HAVE_POSIX_FALLOCATE 1
 #cmakedefine HAVE_SIGACTION 1
 #cmakedefine HAVE_SA_SIGACTION 1
 #cmakedefine HAVE_SETJMP 1
diff --git a/src/video/wayland/SDL_waylandmouse.c b/src/video/wayland/SDL_waylandmouse.c
index 59570e22e6307..178317cb7ceda 100644
--- a/src/video/wayland/SDL_waylandmouse.c
+++ b/src/video/wayland/SDL_waylandmouse.c
@@ -23,11 +23,12 @@
 
 #ifdef SDL_VIDEO_DRIVER_WAYLAND
 
-#include <sys/types.h>
 #include <sys/mman.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <limits.h>
+#include <signal.h>
+#include <errno.h>
 
 #include "../SDL_sysvideo.h"
 
@@ -58,6 +59,7 @@ typedef struct
      */
     SDL_SystemCursor system_cursor;
     void *shm_data;
+    size_t shm_data_size;
 } Wayland_CursorData;
 
 #ifdef SDL_USE_LIBDBUS
@@ -289,27 +291,72 @@ static SDL_bool wayland_get_system_cursor(SDL_VideoData *vdata, Wayland_CursorDa
     return SDL_TRUE;
 }
 
-static int wayland_create_tmp_file(off_t size)
+static int set_tmp_file_size(int fd, off_t size)
 {
-    static const char template[] = "/sdl-shared-XXXXXX";
-    char *xdg_path;
-    char tmp_path[PATH_MAX];
-    int fd;
+#ifdef HAVE_POSIX_FALLOCATE
+    sigset_t set, old_set;
+    int ret;
 
-    xdg_path = SDL_getenv("XDG_RUNTIME_DIR");
-    if (!xdg_path) {
-        return -1;
-    }
+    /* SIGALRM can potentially block a large posix_fallocate() operation
+     * from succeeding, so block it.
+     */
+    sigemptyset(&set);
+    sigaddset(&set, SIGALRM);
+    sigprocmask(SIG_BLOCK, &set, &old_set);
+
+    do {
+        ret = posix_fallocate(fd, 0, size);
+    } while (ret == EINTR);
 
-    SDL_strlcpy(tmp_path, xdg_path, PATH_MAX);
-    SDL_strlcat(tmp_path, template, PATH_MAX);
+    sigprocmask(SIG_SETMASK, &old_set, NULL);
 
-    fd = mkostemp(tmp_path, O_CLOEXEC);
-    if (fd < 0) {
+    if (ret == 0) {
+        return 0;
+    }
+    else if (ret != EINVAL && errno != EOPNOTSUPP) {
         return -1;
     }
+#endif
 
     if (ftruncate(fd, size) < 0) {
+        return -1;
+    }
+    return 0;
+}
+
+static int wayland_create_tmp_file(off_t size)
+{
+    int fd;
+
+#ifdef HAVE_MEMFD_CREATE
+    fd = memfd_create("SDL", MFD_CLOEXEC | MFD_ALLOW_SEALING);
+    if (fd >= 0) {
+        fcntl(fd, F_ADD_SEALS, F_SEAL_SHRINK | F_SEAL_SEAL);
+    } else
+#endif
+    {
+        static const char template[] = "/sdl-shared-XXXXXX";
+        char *xdg_path;
+        char tmp_path[PATH_MAX];
+
+        xdg_path = SDL_getenv("XDG_RUNTIME_DIR");
+        if (!xdg_path) {
+            return -1;
+        }
+
+        SDL_strlcpy(tmp_path, xdg_path, PATH_MAX);
+        SDL_strlcat(tmp_path, template, PATH_MAX);
+
+        fd = mkostemp(tmp_path, O_CLOEXEC);
+        if (fd < 0) {
+            return -1;
+        }
+
+        /* Need to manually unlink the temp files, or they can persist after close and fill up the temp storage. */
+        unlink(tmp_path);
+    }
+
+    if (set_tmp_file_size(fd, size) < 0) {
         close(fd);
         return -1;
     }
@@ -335,17 +382,17 @@ static int create_buffer_from_shm(Wayland_CursorData *d,
     struct wl_shm_pool *shm_pool;
 
     int stride = width * 4;
-    int size = stride * height;
+    d->shm_data_size = stride * height;
 
     int shm_fd;
 
-    shm_fd = wayland_create_tmp_file(size);
+    shm_fd = wayland_create_tmp_file(d->shm_data_size);
     if (shm_fd < 0) {
         return SDL_SetError("Creating mouse cursor buffer failed.");
     }
 
     d->shm_data = mmap(NULL,
-                       size,
+                       d->shm_data_size,
                        PROT_READ | PROT_WRITE,
                        MAP_SHARED,
                        shm_fd,
@@ -358,7 +405,7 @@ static int create_buffer_from_shm(Wayland_CursorData *d,
 
     SDL_assert(d->shm_data != NULL);
 
-    shm_pool = wl_shm_create_pool(data->shm, shm_fd, size);
+    shm_pool = wl_shm_create_pool(data->shm, shm_fd, d->shm_data_size);
     d->buffer = wl_shm_pool_create_buffer(shm_pool,
                                           0,
                                           width,
@@ -459,6 +506,7 @@ static void Wayland_FreeCursorData(Wayland_CursorData *d)
     if (d->buffer) {
         if (d->shm_data) {
             wl_buffer_destroy(d->buffer);
+            munmap(d->shm_data, d->shm_data_size);
         }
         d->buffer = NULL;
     }
@@ -482,7 +530,6 @@ static void Wayland_FreeCursor(SDL_Cursor *cursor)
 
     Wayland_FreeCursorData((Wayland_CursorData *)cursor->driverdata);
 
-    /* Not sure what's meant to happen to shm_data */
     SDL_free(cursor->driverdata);
     SDL_free(cursor);
 }