sdl2-compat: Handle distinction between full-screen and full-screen desktop

From c6988ae4e1448d40c201c3203660cd4599edc078 Mon Sep 17 00:00:00 2001
From: Cameron Gutman <[EMAIL REDACTED]>
Date: Sun, 31 Mar 2024 00:22:49 -0500
Subject: [PATCH] Handle distinction between full-screen and full-screen
 desktop

The existing code would always enter full-screen exclusive mode if
SDL_SetWindowDisplayMode() was called before SDL_SetWindowFullscreen(),
even if the flag value was SDL_WINDOW_FULLSCREEN_DESKTOP.

Likewise, the code would always enter full-screen desktop mode if
SDL_SetWindowDisplayMode() wasn't called, even if the caller passed
SDL_WINDOW_FULLSCREEN to ask for full-screen exclusive mode.

Since SDL3 uses the presence of a full-screen mode to tell whether to
use full-screen exclusive or full-screen desktop mode, we will stash
the provided display mode in a property until we know whether the
caller actually wants modesetting or not.
---
 src/sdl2_compat.c | 81 ++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 63 insertions(+), 18 deletions(-)

diff --git a/src/sdl2_compat.c b/src/sdl2_compat.c
index c457142..6e42d5c 100644
--- a/src/sdl2_compat.c
+++ b/src/sdl2_compat.c
@@ -5718,6 +5718,36 @@ SDL_GetDesktopDisplayMode(int displayIndex, SDL2_DisplayMode *mode)
     return 0;
 }
 
+#define PROP_WINDOW_FULLSCREEN_MODE "sdl2-compat.window.fullscreen-mode"
+
+static int
+ApplyFullscreenMode(SDL_Window *window)
+{
+    SDL_DisplayMode *mode = (SDL_DisplayMode *) SDL3_GetProperty(SDL3_GetWindowProperties(window), PROP_WINDOW_FULLSCREEN_MODE, NULL);
+    if (mode) {
+        /* Always try to enter fullscreen on the current display */
+        mode->displayID = SDL3_GetDisplayForWindow(window);
+    }
+    if (mode && SDL3_SetWindowFullscreenMode(window, mode) == 0) {
+        return 0;
+    } else {
+        int count = 0;
+        const SDL_DisplayMode **list;
+        int ret;
+
+        /* FIXME: at least set a valid fullscreen mode */
+        list = SDL3_GetFullscreenDisplayModes(SDL3_GetPrimaryDisplay(), &count);
+        if (list && count) {
+            ret = SDL3_SetWindowFullscreenMode(window, list[0]);
+        } else {
+            /* If no exclusive modes, use the fullscreen desktop mode. */
+            ret = SDL3_SetWindowFullscreenMode(window, NULL);
+        }
+        SDL3_free((void *)list);
+        return ret;
+    }
+}
+
 DECLSPEC int SDLCALL
 SDL_GetWindowDisplayMode(SDL_Window *window, SDL2_DisplayMode *mode)
 {
@@ -5732,7 +5762,7 @@ SDL_GetWindowDisplayMode(SDL_Window *window, SDL2_DisplayMode *mode)
         return SDL3_InvalidParamError("mode");
     }
 
-    dp = SDL3_GetWindowFullscreenMode(window);
+    dp = (SDL_DisplayMode *) SDL3_GetProperty(SDL3_GetWindowProperties(window), PROP_WINDOW_FULLSCREEN_MODE, (void *) SDL3_GetWindowFullscreenMode(window));
     if (dp) {
         DisplayMode_3to2(dp, mode);
     } else {
@@ -5789,29 +5819,32 @@ SDL_GetClosestDisplayMode(int displayIndex, const SDL2_DisplayMode *mode, SDL2_D
 DECLSPEC int SDLCALL
 SDL_SetWindowDisplayMode(SDL_Window *window, const SDL2_DisplayMode *mode)
 {
+    int ret;
     SDL_DisplayMode dp;
+
+    SDL3_zero(dp);
     DisplayMode_2to3(mode, &dp);
-    dp.displayID = SDL3_GetDisplayForWindow(window);
-    if (SDL3_SetWindowFullscreenMode(window, mode ? &dp : NULL) == 0) {
-        return 0;
-    } else {
-        int count = 0;
-        const SDL_DisplayMode **list;
-        int ret = -1;
 
-        /* FIXME: at least set a valid fullscreen mode */
-        list = SDL3_GetFullscreenDisplayModes(SDL3_GetPrimaryDisplay(), &count);
-        if (list && count) {
-            ret = SDL3_SetWindowFullscreenMode(window, list[0]);
+    /* Store the requested mode in case we aren't in full-screen exclusive mode yet (or ever) */
+    if (mode) {
+        SDL_DisplayMode *property = (SDL_DisplayMode *) SDL3_malloc(sizeof(*property));
+        if (property) {
+            SDL3_copyp(property, &dp);
+            ret = SDL3_SetPropertyWithCleanup(SDL3_GetWindowProperties(window), PROP_WINDOW_FULLSCREEN_MODE, property, CleanupFreeableProperty, NULL);
         } else {
-            /* If no exclusive modes, use the fullscreen desktop mode. */
-            ret = SDL3_SetWindowFullscreenMode(window, NULL);
+            ret = -1;
         }
-        SDL3_free((void *)list);
-        return ret;
+    } else {
+        ret = SDL3_SetProperty(SDL3_GetWindowProperties(window), PROP_WINDOW_FULLSCREEN_MODE, NULL);
     }
-}
 
+    /* If're we're in full-screen exclusive mode now, apply the new display mode */
+    if ((SDL3_GetWindowFlags(window) & SDL_WINDOW_FULLSCREEN) && SDL3_GetWindowFullscreenMode(window)) {
+        ret = ApplyFullscreenMode(window);
+    }
+
+    return ret;
+}
 
 /* this came out of SDL2 directly. */
 static SDL_bool SDL_Vulkan_GetInstanceExtensions_Helper(unsigned int *userCount,
@@ -6254,7 +6287,19 @@ SDL_CreateWindowFrom(const void *data)
 DECLSPEC int SDLCALL
 SDL_SetWindowFullscreen(SDL_Window *window, Uint32 flags)
 {
-    return SDL3_SetWindowFullscreen(window, (flags & SDL2_WINDOW_FULLSCREEN_DESKTOP) != 0);
+    int ret = 0;
+
+    if (flags == SDL2_WINDOW_FULLSCREEN_DESKTOP) {
+        ret = SDL3_SetWindowFullscreenMode(window, NULL);
+    } else if (flags == SDL_WINDOW_FULLSCREEN) {
+        ret = ApplyFullscreenMode(window);
+    }
+
+    if (ret == 0) {
+        ret = SDL3_SetWindowFullscreen(window, (flags & SDL2_WINDOW_FULLSCREEN_DESKTOP) != 0);
+    }
+
+    return ret;
 }
 
 DECLSPEC void SDLCALL