SDL: wayland: Use the cached window size when switching from non-floating to floating window state

From fe396e306effb170b1abd093111d3f5c5338ae4b Mon Sep 17 00:00:00 2001
From: Frank Praznik <[EMAIL REDACTED]>
Date: Sat, 19 Nov 2022 11:28:31 -0500
Subject: [PATCH] wayland: Use the cached window size when switching from
 non-floating to floating window state

When changing the window state from non-floating to floating (e.g. leaving fullscreen), libdecor can send bogus content sizes that are +/- the height of the window title bar and start 'walking' the window height in one direction or the other with every transition.

The floating window size is known, so use the cached value instead of the size reported by libdecor when restoring the floating state.
---
 src/video/wayland/SDL_waylandwindow.c | 16 ++++++++++++----
 src/video/wayland/SDL_waylandwindow.h |  1 +
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/video/wayland/SDL_waylandwindow.c b/src/video/wayland/SDL_waylandwindow.c
index a329ad357b3f..a31f6baf6e9d 100644
--- a/src/video/wayland/SDL_waylandwindow.c
+++ b/src/video/wayland/SDL_waylandwindow.c
@@ -878,11 +878,17 @@ decoration_frame_configure(struct libdecor_frame *frame,
         wind->floating_resize_pending = SDL_FALSE;
     } else {
         /*
-         * XXX: When hiding a floating window, libdecor can send bogus content sizes that
-         *      are +/- the height of the title bar, which distorts the window size.
-         *      Ignore any values from libdecor when hiding a floating window.
+         * XXX: libdecor can send bogus content sizes that are +/- the height
+         *      of the title bar when hiding a window or transitioning from
+         *      non-floating to floating state, which distorts the window size.
+         *
+         *      Ignore any size values from libdecor in these scenarios in
+         *      favor of the cached window size.
+         *
+         *      https://gitlab.gnome.org/jadahl/libdecor/-/issues/40
          */
-        const SDL_bool use_cached_size = (window->is_hiding || !!(window->flags & SDL_WINDOW_HIDDEN));
+        const SDL_bool use_cached_size = (floating && !wind->was_floating) ||
+                                         (window->is_hiding || !!(window->flags & SDL_WINDOW_HIDDEN));
 
         /* This will never set 0 for width/height unless the function returns false */
         if (use_cached_size || !libdecor_configuration_get_content_size(configuration, frame, &width, &height)) {
@@ -905,6 +911,8 @@ decoration_frame_configure(struct libdecor_frame *frame,
         wind->floating_height = height;
     }
 
+    wind->was_floating = floating;
+
     /* Do the resize on the SDL side (this will set window->w/h)... */
     Wayland_HandleResize(window, width, height, scale_factor);
     wind->shell_surface.libdecor.initial_configure_seen = SDL_TRUE;
diff --git a/src/video/wayland/SDL_waylandwindow.h b/src/video/wayland/SDL_waylandwindow.h
index b6a3aa28c36f..6e270104877e 100644
--- a/src/video/wayland/SDL_waylandwindow.h
+++ b/src/video/wayland/SDL_waylandwindow.h
@@ -102,6 +102,7 @@ typedef struct {
     int window_width, window_height;
     SDL_bool needs_resize_event;
     SDL_bool floating_resize_pending;
+    SDL_bool was_floating;
     SDL_bool is_fullscreen;
     SDL_bool in_fullscreen_transition;
     Uint32 fullscreen_flags;