SDL: wayland: Always commit window constraints before entering fullscreen

From 4a3277b0f9878d6e66e60998a26115f8e2b6e453 Mon Sep 17 00:00:00 2001
From: Frank Praznik <[EMAIL REDACTED]>
Date: Thu, 19 May 2022 15:13:02 -0400
Subject: [PATCH] wayland: Always commit window constraints before entering
 fullscreen

XDG-toplevel min/max size values are double-buffered data and must be committed before entering the fullscreen state, or a max window size value smaller than the display dimensions may cause the compositor to incorrectly configure the fullscreen window size.  This fixes windowed->fullscreen transitions on GNOME, where, previously, certain combinations of window flags and min/max size values could cause entering fullscreen mode to fail with odd window sizes and/or offsets due to the new max size values not being committed before entering fullscreen, causing the compositor to clamp to the old values.

In the case of libdecor, it has its own layer of buffering on top of the xdg-toplevel surface for the min/max window dimensions, so both a frame commit and surface commit are required to set the state properly.
---
 src/video/wayland/SDL_waylandwindow.c | 29 +++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/src/video/wayland/SDL_waylandwindow.c b/src/video/wayland/SDL_waylandwindow.c
index 8be9cd9874e..cfd835b764e 100644
--- a/src/video/wayland/SDL_waylandwindow.c
+++ b/src/video/wayland/SDL_waylandwindow.c
@@ -351,6 +351,13 @@ SetMinMaxDimensions(SDL_Window *window, SDL_bool commit)
         libdecor_frame_set_max_content_size(wind->shell_surface.libdecor.frame,
                                             max_width,
                                             max_height);
+
+        if (commit) {
+            struct libdecor_state *state = libdecor_state_new(GetWindowWidth(window), GetWindowHeight(window));
+            libdecor_frame_commit(wind->shell_surface.libdecor.frame, state, NULL);
+            libdecor_state_free(state);
+            wl_surface_commit(wind->surface);
+        }
     } else
 #endif
     if (viddata->shell.xdg) {
@@ -396,17 +403,31 @@ SetFullscreen(SDL_Window *window, struct wl_output *output, SDL_bool commit)
         }
         if (output) {
             if (!(window->flags & SDL_WINDOW_RESIZABLE)) {
-                /* ensure that window is resizable before going into fullscreen */
+                /* Ensure that window is resizable before going into fullscreen.
+                 * This triggers a frame commit internally, so a separate one is not necessary.
+                 */
                 libdecor_frame_set_capabilities(wind->shell_surface.libdecor.frame, LIBDECOR_ACTION_RESIZE);
                 wl_surface_commit(wind->surface);
+            } else if (commit) {
+                struct libdecor_state *state = libdecor_state_new(GetWindowWidth(window), GetWindowHeight(window));
+                libdecor_frame_commit(wind->shell_surface.libdecor.frame, state, NULL);
+                libdecor_state_free(state);
+                wl_surface_commit(wind->surface);
             }
+
             libdecor_frame_set_fullscreen(wind->shell_surface.libdecor.frame, output);
         } else {
             libdecor_frame_unset_fullscreen(wind->shell_surface.libdecor.frame);
+
             if (!(window->flags & SDL_WINDOW_RESIZABLE)) {
                 /* restore previous RESIZE capability */
                 libdecor_frame_unset_capabilities(wind->shell_surface.libdecor.frame, LIBDECOR_ACTION_RESIZE);
                 wl_surface_commit(wind->surface);
+            } else if (commit) {
+                struct libdecor_state *state = libdecor_state_new(GetWindowWidth(window), GetWindowHeight(window));
+                libdecor_frame_commit(wind->shell_surface.libdecor.frame, state, NULL);
+                libdecor_state_free(state);
+                wl_surface_commit(wind->surface);
             }
         }
     } else
@@ -415,14 +436,14 @@ SetFullscreen(SDL_Window *window, struct wl_output *output, SDL_bool commit)
         if (wind->shell_surface.xdg.roleobj.toplevel == NULL) {
             return; /* Can't do anything yet, wait for ShowWindow */
         }
+        if (commit) {
+            wl_surface_commit(wind->surface);
+        }
         if (output) {
             xdg_toplevel_set_fullscreen(wind->shell_surface.xdg.roleobj.toplevel, output);
         } else {
             xdg_toplevel_unset_fullscreen(wind->shell_surface.xdg.roleobj.toplevel);
         }
-        if (commit) {
-            wl_surface_commit(wind->surface);
-        }
     }
 }