SDL: Add support for modal windows to more platforms

From c6a70d68989cc08e1d0785627bcced5e59818aeb Mon Sep 17 00:00:00 2001
From: Semphris <[EMAIL REDACTED]>
Date: Wed, 3 Apr 2024 16:44:03 -0400
Subject: [PATCH] Add support for modal windows to more platforms

- Adds support for modal windows to Win32, Mac, and Haiku, and enhances functionality on Wayland and X11, which previous set only the parent window, but not the modal state.
- Windows can be declared modal at creation time, and the modal state can be toggled at any time via SDL_SetWindowModalFor() (tested with UE5 through sdl2-compat).
- Allows dynamic unparenting/reparenting of windows.
- Includes a modal window test.
---
 include/SDL3/SDL_video.h              |  17 ++-
 src/video/SDL_video.c                 | 104 ++++++++++++----
 src/video/cocoa/SDL_cocoaevents.m     |   8 ++
 src/video/cocoa/SDL_cocoavideo.m      |   1 +
 src/video/cocoa/SDL_cocoawindow.h     |   2 +
 src/video/cocoa/SDL_cocoawindow.m     |  27 ++++
 src/video/haiku/SDL_bwindow.cc        |  23 +++-
 src/video/wayland/SDL_waylandvideo.c  |   8 ++
 src/video/wayland/SDL_waylandvideo.h  |   1 +
 src/video/wayland/SDL_waylandwindow.c |  74 +++++++----
 src/video/wayland/SDL_waylandwindow.h |   2 +
 src/video/windows/SDL_windowsvideo.c  |   1 +
 src/video/windows/SDL_windowswindow.c |  44 +++++++
 src/video/windows/SDL_windowswindow.h |   1 +
 src/video/x11/SDL_x11video.c          |   2 +
 src/video/x11/SDL_x11video.h          |   2 +
 src/video/x11/SDL_x11window.c         |  43 ++++++-
 test/CMakeLists.txt                   |   1 +
 test/testmodal.c                      | 172 ++++++++++++++++++++++++++
 wayland-protocols/xdg-dialog-v1.xml   | 110 ++++++++++++++++
 20 files changed, 592 insertions(+), 51 deletions(-)
 create mode 100644 test/testmodal.c
 create mode 100644 wayland-protocols/xdg-dialog-v1.xml

diff --git a/include/SDL3/SDL_video.h b/include/SDL3/SDL_video.h
index 0bf26e496fb59..38d76f296444f 100644
--- a/include/SDL3/SDL_video.h
+++ b/include/SDL3/SDL_video.h
@@ -144,6 +144,7 @@ typedef Uint32 SDL_WindowFlags;
 #define SDL_WINDOW_INPUT_FOCUS          0x00000200U /**< window has input focus */
 #define SDL_WINDOW_MOUSE_FOCUS          0x00000400U /**< window has mouse focus */
 #define SDL_WINDOW_EXTERNAL             0x00000800U /**< window not created by SDL */
+#define SDL_WINDOW_MODAL                0x00001000U /**< window is modal */
 #define SDL_WINDOW_HIGH_PIXEL_DENSITY   0x00002000U /**< window uses high pixel density back buffer if possible */
 #define SDL_WINDOW_MOUSE_CAPTURE        0x00004000U /**< window has mouse captured (unrelated to MOUSE_GRABBED) */
 #define SDL_WINDOW_ALWAYS_ON_TOP        0x00008000U /**< window should always be above others */
@@ -907,13 +908,15 @@ extern DECLSPEC SDL_Window *SDLCALL SDL_CreatePopupWindow(SDL_Window *parent, in
  *   with Metal rendering
  * - `SDL_PROP_WINDOW_CREATE_MINIMIZED_BOOLEAN`: true if the window should
  *   start minimized
+ * - `SDL_PROP_WINDOW_CREATE_MODAL_BOOLEAN`: true if the window is modal to its
+ *   parent
  * - `SDL_PROP_WINDOW_CREATE_MOUSE_GRABBED_BOOLEAN`: true if the window starts
  *   with grabbed mouse focus
  * - `SDL_PROP_WINDOW_CREATE_OPENGL_BOOLEAN`: true if the window will be used
  *   with OpenGL rendering
  * - `SDL_PROP_WINDOW_CREATE_PARENT_POINTER`: an SDL_Window that will be the
- *   parent of this window, required for windows with the "toolip" and "menu"
- *   properties
+ *   parent of this window, required for windows with the "toolip", "menu", and
+ *   "modal" properties
  * - `SDL_PROP_WINDOW_CREATE_RESIZABLE_BOOLEAN`: true if the window should be
  *   resizable
  * - `SDL_PROP_WINDOW_CREATE_TITLE_STRING`: the title of the window, in UTF-8
@@ -1008,6 +1011,7 @@ extern DECLSPEC SDL_Window *SDLCALL SDL_CreateWindowWithProperties(SDL_Propertie
 #define SDL_PROP_WINDOW_CREATE_MENU_BOOLEAN                        "menu"
 #define SDL_PROP_WINDOW_CREATE_METAL_BOOLEAN                       "metal"
 #define SDL_PROP_WINDOW_CREATE_MINIMIZED_BOOLEAN                   "minimized"
+#define SDL_PROP_WINDOW_CREATE_MODAL_BOOLEAN                       "modal"
 #define SDL_PROP_WINDOW_CREATE_MOUSE_GRABBED_BOOLEAN               "mouse_grabbed"
 #define SDL_PROP_WINDOW_CREATE_OPENGL_BOOLEAN                      "opengl"
 #define SDL_PROP_WINDOW_CREATE_PARENT_POINTER                      "parent"
@@ -2000,7 +2004,12 @@ extern DECLSPEC int SDLCALL SDL_SetWindowOpacity(SDL_Window *window, float opaci
 extern DECLSPEC int SDLCALL SDL_GetWindowOpacity(SDL_Window *window, float *out_opacity);
 
 /**
- * Set the window as a modal for another window.
+ * Set the window as a modal to a parent window.
+ *
+ * If the window is already modal to an existing window, it will be reparented to the new owner.
+ * Setting the parent window to null unparents the modal window and removes modal status.
+ *
+ * Setting a window as modal to a parent that is a descendent of the modal window results in undefined behavior.
  *
  * \param modal_window the window that should be set modal
  * \param parent_window the parent window for the modal window
@@ -2181,6 +2190,8 @@ extern DECLSPEC int SDLCALL SDL_FlashWindow(SDL_Window *window, SDL_FlashOperati
 /**
  * Destroy a window.
  *
+ * Any popups or modal windows owned by the window will be recursively destroyed as well.
+ *
  * If `window` is NULL, this function will return immediately after setting
  * the SDL error message to "Invalid window". See SDL_GetError().
  *
diff --git a/src/video/SDL_video.c b/src/video/SDL_video.c
index 347266bc0e334..18e4d255abed3 100644
--- a/src/video/SDL_video.c
+++ b/src/video/SDL_video.c
@@ -200,6 +200,33 @@ static void SDL_SyncIfRequired(SDL_Window *window)
     }
 }
 
+static void SDL_SetWindowParent(SDL_Window *window, SDL_Window *parent)
+{
+    /* Unlink the window from the existing parent. */
+    if (window->parent) {
+        if (window->next_sibling) {
+            window->next_sibling->prev_sibling = window->prev_sibling;
+        }
+        if (window->prev_sibling) {
+            window->prev_sibling->next_sibling = window->next_sibling;
+        } else {
+            window->parent->first_child = window->next_sibling;
+        }
+
+        window->parent = NULL;
+    }
+
+    if (parent) {
+        window->parent = parent;
+
+        window->next_sibling = parent->first_child;
+        if (parent->first_child) {
+            parent->first_child->prev_sibling = window;
+        }
+        parent->first_child = window;
+    }
+}
+
 /* Support for framebuffer emulation using an accelerated renderer */
 
 #define SDL_PROP_WINDOW_TEXTUREDATA_POINTER "SDL.internal.window.texturedata"
@@ -2002,6 +2029,7 @@ static struct {
     { SDL_PROP_WINDOW_CREATE_MENU_BOOLEAN,               SDL_WINDOW_POPUP_MENU,          SDL_FALSE },
     { SDL_PROP_WINDOW_CREATE_METAL_BOOLEAN,              SDL_WINDOW_METAL,               SDL_FALSE },
     { SDL_PROP_WINDOW_CREATE_MINIMIZED_BOOLEAN,          SDL_WINDOW_MINIMIZED,           SDL_FALSE },
+    { SDL_PROP_WINDOW_CREATE_MODAL_BOOLEAN,              SDL_WINDOW_MODAL,               SDL_FALSE },
     { SDL_PROP_WINDOW_CREATE_MOUSE_GRABBED_BOOLEAN,      SDL_WINDOW_MOUSE_GRABBED,       SDL_FALSE },
     { SDL_PROP_WINDOW_CREATE_OPENGL_BOOLEAN,             SDL_WINDOW_OPENGL,              SDL_FALSE },
     { SDL_PROP_WINDOW_CREATE_RESIZABLE_BOOLEAN,          SDL_WINDOW_RESIZABLE,           SDL_FALSE },
@@ -2057,6 +2085,11 @@ SDL_Window *SDL_CreateWindowWithProperties(SDL_PropertiesID props)
         }
     }
 
+    if ((flags & SDL_WINDOW_MODAL) && (!parent || parent->magic != &_this->window_magic)) {
+        SDL_SetError("Modal windows must specify a parent window");
+        return NULL;
+    }
+
     if ((flags & (SDL_WINDOW_TOOLTIP | SDL_WINDOW_POPUP_MENU)) != 0) {
         if (!(_this->device_caps & VIDEO_DEVICE_CAPS_HAS_POPUP_WINDOW_SUPPORT)) {
             SDL_Unsupported();
@@ -2074,7 +2107,7 @@ SDL_Window *SDL_CreateWindowWithProperties(SDL_PropertiesID props)
     }
 
     /* Ensure no more than one of these flags is set */
-    type_flags = flags & (SDL_WINDOW_UTILITY | SDL_WINDOW_TOOLTIP | SDL_WINDOW_POPUP_MENU);
+    type_flags = flags & (SDL_WINDOW_UTILITY | SDL_WINDOW_TOOLTIP | SDL_WINDOW_POPUP_MENU | SDL_WINDOW_MODAL);
     if (type_flags & (type_flags - 1)) {
         SDL_SetError("Conflicting window type flags specified: 0x%.8x", (unsigned int)type_flags);
         return NULL;
@@ -2200,14 +2233,9 @@ SDL_Window *SDL_CreateWindowWithProperties(SDL_PropertiesID props)
     }
     _this->windows = window;
 
-    if (parent) {
-        window->parent = parent;
-
-        window->next_sibling = parent->first_child;
-        if (parent->first_child) {
-            parent->first_child->prev_sibling = window;
-        }
-        parent->first_child = window;
+    /* Set the parent before creation if this is non-modal, otherwise it will be set later. */
+    if (!(flags & SDL_WINDOW_MODAL)) {
+        SDL_SetWindowParent(window, parent);
     }
 
     if (_this->CreateSDLWindow && _this->CreateSDLWindow(_this, window, props) < 0) {
@@ -2236,6 +2264,9 @@ SDL_Window *SDL_CreateWindowWithProperties(SDL_PropertiesID props)
     flags = window->flags;
 #endif
 
+    if (flags & SDL_WINDOW_MODAL) {
+        SDL_SetWindowModalFor(window, parent);
+    }
     if (title) {
         SDL_SetWindowTitle(window, title);
     }
@@ -2293,6 +2324,7 @@ int SDL_RecreateWindow(SDL_Window *window, SDL_WindowFlags flags)
     SDL_bool need_vulkan_unload = SDL_FALSE;
     SDL_bool need_vulkan_load = SDL_FALSE;
     SDL_WindowFlags graphics_flags;
+    SDL_Window *parent = window->parent;
 
     /* ensure no more than one of these flags is set */
     graphics_flags = flags & (SDL_WINDOW_OPENGL | SDL_WINDOW_METAL | SDL_WINDOW_VULKAN);
@@ -2317,6 +2349,11 @@ int SDL_RecreateWindow(SDL_Window *window, SDL_WindowFlags flags)
         flags &= ~SDL_WINDOW_EXTERNAL;
     }
 
+    /* If this is a modal dialog, clear the modal status. */
+    if (window->flags & SDL_WINDOW_MODAL) {
+        SDL_SetWindowModalFor(window, NULL);
+    }
+
     /* Restore video mode, etc. */
     if (!(window->flags & SDL_WINDOW_EXTERNAL)) {
         const SDL_bool restore_on_show = window->restore_on_show;
@@ -2410,6 +2447,10 @@ int SDL_RecreateWindow(SDL_Window *window, SDL_WindowFlags flags)
         window->flags |= SDL_WINDOW_EXTERNAL;
     }
 
+    if (flags & SDL_WINDOW_MODAL) {
+        SDL_SetWindowModalFor(window, parent);
+    }
+
     if (_this->SetWindowTitle && window->title) {
         _this->SetWindowTitle(_this, window);
     }
@@ -3259,15 +3300,35 @@ int SDL_GetWindowOpacity(SDL_Window *window, float *out_opacity)
 int SDL_SetWindowModalFor(SDL_Window *modal_window, SDL_Window *parent_window)
 {
     CHECK_WINDOW_MAGIC(modal_window, -1);
-    CHECK_WINDOW_MAGIC(parent_window, -1);
     CHECK_WINDOW_NOT_POPUP(modal_window, -1);
-    CHECK_WINDOW_NOT_POPUP(parent_window, -1);
+
+    if (parent_window) {
+        CHECK_WINDOW_MAGIC(parent_window, -1);
+        CHECK_WINDOW_NOT_POPUP(parent_window, -1);
+    }
 
     if (!_this->SetWindowModalFor) {
         return SDL_Unsupported();
     }
 
-    return _this->SetWindowModalFor(_this, modal_window, parent_window);
+    if (parent_window) {
+        modal_window->flags |= SDL_WINDOW_MODAL;
+    } else if (modal_window->flags & SDL_WINDOW_MODAL) {
+        modal_window->flags &= ~SDL_WINDOW_MODAL;
+    } else {
+        return 0; /* Not modal; nothing to do. */
+    }
+
+    const int ret = _this->SetWindowModalFor(_this, modal_window, parent_window);
+
+    /* The existing parent might be needed when changing the modal status,
+     * so don't change the heirarchy until after setting the new modal state.
+     */
+    if (!ret) {
+        SDL_SetWindowParent(modal_window, !ret ? parent_window : NULL);
+    }
+
+    return ret;
 }
 
 int SDL_SetWindowInputFocus(SDL_Window *window)
@@ -3686,16 +3747,12 @@ void SDL_DestroyWindow(SDL_Window *window)
 
     SDL_DestroyProperties(window->props);
 
-    /* If this is a child window, unlink it from its siblings */
-    if (window->parent) {
-        if (window->next_sibling) {
-            window->next_sibling->prev_sibling = window->prev_sibling;
-        }
-        if (window->prev_sibling) {
-            window->prev_sibling->next_sibling = window->next_sibling;
-        } else {
-            window->parent->first_child = window->next_sibling;
-        }
+    /* Clear the modal status, but don't unset the parent, as it may be
+     * needed later in the destruction process if a backend needs to
+     * update the input focus.
+     */
+    if (_this->SetWindowModalFor && (window->flags & SDL_WINDOW_MODAL)) {
+        _this->SetWindowModalFor(_this, window, NULL);
     }
 
     /* Restore video mode, etc. */
@@ -3765,6 +3822,9 @@ void SDL_DestroyWindow(SDL_Window *window)
     SDL_free(window->title);
     SDL_DestroySurface(window->icon);
 
+    /* Unlink the window from its siblings. */
+    SDL_SetWindowParent(window, NULL);
+
     /* Unlink the window from the list */
     if (window->next) {
         window->next->prev = window->prev;
diff --git a/src/video/cocoa/SDL_cocoaevents.m b/src/video/cocoa/SDL_cocoaevents.m
index 81f086cea30d0..7bd0a54aed3a4 100644
--- a/src/video/cocoa/SDL_cocoaevents.m
+++ b/src/video/cocoa/SDL_cocoaevents.m
@@ -563,6 +563,14 @@ Uint64 Cocoa_GetEventTimestamp(NSTimeInterval nsTimestamp)
 
 int Cocoa_PumpEventsUntilDate(SDL_VideoDevice *_this, NSDate *expiration, bool accumulate)
 {
+    /* Run any existing modal sessions. */
+    for (SDL_Window *w = _this->windows; w; w = w->next) {
+        SDL_CocoaWindowData *data = (__bridge SDL_CocoaWindowData *)w->driverdata;
+        if (data.modal_session) {
+            [NSApp runModalSession:data.modal_session];
+        }
+    }
+
     for (;;) {
         NSEvent *event = [NSApp nextEventMatchingMask:NSEventMaskAny untilDate:expiration inMode:NSDefaultRunLoopMode dequeue:YES];
         if (event == nil) {
diff --git a/src/video/cocoa/SDL_cocoavideo.m b/src/video/cocoa/SDL_cocoavideo.m
index ae4bbcfbf016a..385d0eb978205 100644
--- a/src/video/cocoa/SDL_cocoavideo.m
+++ b/src/video/cocoa/SDL_cocoavideo.m
@@ -122,6 +122,7 @@ static void Cocoa_DeleteDevice(SDL_VideoDevice *device)
         device->UpdateWindowShape = Cocoa_UpdateWindowShape;
         device->FlashWindow = Cocoa_FlashWindow;
         device->SetWindowFocusable = Cocoa_SetWindowFocusable;
+        device->SetWindowModalFor = Cocoa_SetWindowModalFor;
         device->SyncWindow = Cocoa_SyncWindow;
 
 #ifdef SDL_VIDEO_OPENGL_CGL
diff --git a/src/video/cocoa/SDL_cocoawindow.h b/src/video/cocoa/SDL_cocoawindow.h
index 4f4d9daccec2b..b4d4cec13191d 100644
--- a/src/video/cocoa/SDL_cocoawindow.h
+++ b/src/video/cocoa/SDL_cocoawindow.h
@@ -138,6 +138,7 @@ typedef enum
 @property(nonatomic) NSInteger flash_request;
 @property(nonatomic) SDL_Window *keyboard_focus;
 @property(nonatomic) Cocoa_WindowListener *listener;
+@property(nonatomic) NSModalSession modal_session;
 @property(nonatomic) SDL_CocoaVideoData *videodata;
 @property(nonatomic) SDL_bool send_floating_size;
 @property(nonatomic) SDL_bool send_floating_position;
@@ -178,6 +179,7 @@ extern int Cocoa_SetWindowHitTest(SDL_Window *window, SDL_bool enabled);
 extern void Cocoa_AcceptDragAndDrop(SDL_Window *window, SDL_bool accept);
 extern int Cocoa_FlashWindow(SDL_VideoDevice *_this, SDL_Window *window, SDL_FlashOperation operation);
 extern int Cocoa_SetWindowFocusable(SDL_VideoDevice *_this, SDL_Window *window, SDL_bool focusable);
+extern int Cocoa_SetWindowModalFor(SDL_VideoDevice *_this, SDL_Window *modal_window, SDL_Window *parent_window);
 extern int Cocoa_SyncWindow(SDL_VideoDevice *_this, SDL_Window *window);
 
 #endif /* SDL_cocoawindow_h_ */
diff --git a/src/video/cocoa/SDL_cocoawindow.m b/src/video/cocoa/SDL_cocoawindow.m
index 13f52ddc63fa6..9b6e6ce48375d 100644
--- a/src/video/cocoa/SDL_cocoawindow.m
+++ b/src/video/cocoa/SDL_cocoawindow.m
@@ -2369,6 +2369,10 @@ void Cocoa_ShowWindow(SDL_VideoDevice *_this, SDL_Window *window)
                 NSWindow *nsparent = ((__bridge SDL_CocoaWindowData *)window->parent->driverdata).nswindow;
                 [nsparent addChildWindow:nswindow ordered:NSWindowAbove];
             } else {
+                if ((window->flags & SDL_WINDOW_MODAL) && window->parent) {
+                    Cocoa_SetWindowModalFor(_this, window, window->parent);
+                }
+
                 if (bActivate) {
                     [nswindow makeKeyAndOrderFront:nil];
                 } else {
@@ -2402,6 +2406,11 @@ void Cocoa_HideWindow(SDL_VideoDevice *_this, SDL_Window *window)
             [nswindow close];
         }
 
+        /* If this window is the source of a modal session, end it when
+         * hidden, or other windows will be prevented from closing.
+         */
+        Cocoa_SetWindowModalFor(_this, window, NULL);
+
         /* Transfer keyboard focus back to the parent */
         if (window->flags & SDL_WINDOW_POPUP_MENU) {
             if (window == SDL_GetKeyboardFocus()) {
@@ -2928,6 +2937,24 @@ void Cocoa_AcceptDragAndDrop(SDL_Window *window, SDL_bool accept)
     }
 }
 
+int Cocoa_SetWindowModalFor(SDL_VideoDevice *_this, SDL_Window *modal_window, SDL_Window *parent_window)
+{
+    @autoreleasepool {
+        SDL_CocoaWindowData *modal_data = (__bridge SDL_CocoaWindowData *)modal_window->driverdata;
+
+        if (modal_data.modal_session) {
+            [NSApp endModalSession:modal_data.modal_session];
+            modal_data.modal_session = nil;
+        }
+
+        if (parent_window) {
+            modal_data.modal_session = [NSApp beginModalSessionForWindow:modal_data.nswindow];
+        }
+    }
+
+    return 0;
+}
+
 int Cocoa_FlashWindow(SDL_VideoDevice *_this, SDL_Window *window, SDL_FlashOperation operation)
 {
     @autoreleasepool {
diff --git a/src/video/haiku/SDL_bwindow.cc b/src/video/haiku/SDL_bwindow.cc
index 86436a0db84e9..b4deb70d075e2 100644
--- a/src/video/haiku/SDL_bwindow.cc
+++ b/src/video/haiku/SDL_bwindow.cc
@@ -39,7 +39,7 @@ static SDL_INLINE SDL_BLooper *_GetBeLooper() {
     return SDL_Looper;
 }
 
-static int _InitWindow(SDL_VideoDevice *_this, SDL_Window *window) {
+static int _InitWindow(SDL_VideoDevice *_this, SDL_Window *window, SDL_PropertiesID create_props) {
     uint32 flags = 0;
     window_look look = B_TITLED_WINDOW_LOOK;
 
@@ -77,7 +77,7 @@ static int _InitWindow(SDL_VideoDevice *_this, SDL_Window *window) {
 }
 
 int HAIKU_CreateWindow(SDL_VideoDevice *_this, SDL_Window *window, SDL_PropertiesID create_props) {
-    if (_InitWindow(_this, window) < 0) {
+    if (_InitWindow(_this, window, create_props) < 0) {
         return -1;
     }
 
@@ -171,6 +171,25 @@ int HAIKU_SetWindowMouseGrab(SDL_VideoDevice *_this, SDL_Window * window, SDL_bo
     return SDL_Unsupported();
 }
 
+int HAIKU_SetWindowModalFor(SDL_VideoDevice *_this, SDL_Window *modal_window, SDL_Window *parent_window) {
+    if (modal_window->parent && modal_window->parent != parent_window) {
+        /* Remove from the subset of a previous parent. */
+        _ToBeWin(modal_window)->RemoveFromSubset(_ToBeWin(modal_window->parent));
+    }
+
+    if (parent_window) {
+        _ToBeWin(modal_window)->SetLook(B_MODAL_WINDOW_LOOK);
+        _ToBeWin(modal_window)->SetFeel(B_MODAL_SUBSET_WINDOW_FEEL);
+        _ToBeWin(modal_window)->AddToSubset(_ToBeWin(parent_window));
+    } else {
+        window_look look = (modal_window->flags & SDL_WINDOW_BORDERLESS) ? B_NO_BORDER_WINDOW_LOOK : B_TITLED_WINDOW_LOOK;
+        _ToBeWin(modal_window)->SetLook(look);
+        _ToBeWin(modal_window)->SetFeel(B_NORMAL_WINDOW_FEEL);
+    }
+
+    return 0;
+}
+
 void HAIKU_DestroyWindow(SDL_VideoDevice *_this, SDL_Window * window) {
     _ToBeWin(window)->LockLooper();    /* This MUST be locked */
     _GetBeLooper()->ClearID(_ToBeWin(window));
diff --git a/src/video/wayland/SDL_waylandvideo.c b/src/video/wayland/SDL_waylandvideo.c
index 7cfb5a25e48cd..173fc7f7c99d1 100644
--- a/src/video/wayland/SDL_waylandvideo.c
+++ b/src/video/wayland/SDL_waylandvideo.c
@@ -57,6 +57,7 @@
 #include "viewporter-client-protocol.h"
 #include "xdg-activation-v1-client-protocol.h"
 #include "xdg-decoration-unstable-v1-client-protocol.h"
+#include "xdg-dialog-v1-client-protocol.h"
 #include "xdg-foreign-unstable-v2-client-protocol.h"
 #include "xdg-output-unstable-v1-client-protocol.h"
 #include "xdg-shell-client-protocol.h"
@@ -1088,6 +1089,8 @@ static void display_handle_global(void *data, struct wl_registry *registry, uint
         }
     } else if (SDL_strcmp(interface, "zxdg_exporter_v2") == 0) {
         d->zxdg_exporter_v2 = wl_registry_bind(d->registry, id, &zxdg_exporter_v2_interface, 1);
+    } else if (SDL_strcmp(interface, "xdg_wm_dialog_v1") == 0) {
+        d->xdg_wm_dialog_v1 = wl_registry_bind(d->registry, id, &xdg_wm_dialog_v1_interface, 1);
     } else if (SDL_strcmp(interface, "kde_output_order_v1") == 0) {
         d->kde_output_order = wl_registry_bind(d->registry, id, &kde_output_order_v1_interface, 1);
         kde_output_order_v1_add_listener(d->kde_output_order, &kde_output_order_listener, d);
@@ -1346,6 +1349,11 @@ static void Wayland_VideoCleanup(SDL_VideoDevice *_this)
         data->zxdg_exporter_v2 = NULL;
     }
 
+    if (data->xdg_wm_dialog_v1) {
+        xdg_wm_dialog_v1_destroy(data->xdg_wm_dialog_v1);
+        data->xdg_wm_dialog_v1 = NULL;
+    }
+
     if (data->kde_output_order) {
         Wayland_FlushOutputOrder(data);
         kde_output_order_v1_destroy(data->kde_output_order);
diff --git a/src/video/wayland/SDL_waylandvideo.h b/src/video/wayland/SDL_waylandvideo.h
index c6ee45788f7f6..7228636edd292 100644
--- a/src/video/wayland/SDL_waylandvideo.h
+++ b/src/video/wayland/SDL_waylandvideo.h
@@ -80,6 +80,7 @@ struct SDL_VideoData
     struct wp_fractional_scale_manager_v1 *fractional_scale_manager;
     struct zwp_input_timestamps_manager_v1 *input_timestamps_manager;
     struct zxdg_exporter_v2 *zxdg_exporter_v2;
+    struct xdg_wm_dialog_v1 *xdg_wm_dialog_v1;
     struct kde_output_order_v1 *kde_output_order;
 
     struct xkb_context *xkb_context;
diff --git a/src/video/wayland/SDL_waylandwindow.c b/src/video/wayland/SDL_waylandwindow.c
index 28d7b2cbbc5b4..83dc22a258de6 100644
--- a/src/video/wayland/SDL_waylandwindow.c
+++ b/src/video/wayland/SDL_waylandwindow.c
@@ -39,6 +39,7 @@
 #include "viewporter-client-protocol.h"
 #include "fractional-scale-v1-client-protocol.h"
 #include "xdg-foreign-unstable-v2-client-protocol.h"
+#include "xdg-dialog-v1-client-protocol.h"
 
 #ifdef HAVE_LIBDECOR_H
 #include <libdecor.h>
@@ -654,6 +655,8 @@ static void surface_frame_done(void *data, struct wl_callback *cb, uint32_t time
         for (SDL_Window *w = wind->sdlwindow->first_child; w; w = w->next_sibling) {
             if (w->driverdata->surface_status == WAYLAND_SURFACE_STATUS_SHOW_PENDING) {
                 Wayland_ShowWindow(SDL_GetVideoDevice(), w);
+            } else if ((w->flags & SDL_WINDOW_MODAL) && w->driverdata->modal_reparenting_required) {
+                Wayland_SetWindowModalFor(SDL_GetVideoDevice(), w, w->parent);
             }
         }
 
@@ -1434,35 +1437,56 @@ int Wayland_SetWindowModalFor(SDL_VideoDevice *_this, SDL_Window *modal_window,
 {
     SDL_VideoData *viddata = _this->driverdata;
     SDL_WindowData *modal_data = modal_window->driverdata;
-    SDL_WindowData *parent_data = parent_window->driverdata;
+    SDL_WindowData *parent_data = parent_window ? parent_window->driverdata : NULL;
+    struct xdg_toplevel *modal_toplevel = NULL;
+    struct xdg_toplevel *parent_toplevel = NULL;
 
-    if (modal_data->shell_surface_type == WAYLAND_SURFACE_XDG_POPUP || parent_data->shell_surface_type == WAYLAND_SURFACE_XDG_POPUP) {
-        return SDL_SetError("Modal/Parent was a popup, not a toplevel");
+    modal_data->modal_reparenting_required = SDL_FALSE;
+
+    if (parent_data && parent_data->surface_status != WAYLAND_SURFACE_STATUS_SHOWN) {
+        /* Need to wait for the parent to become mapped, or it's the same as setting a null parent. */
+        modal_data->modal_reparenting_required = SDL_TRUE;
+        return 0;
     }
 
+    /* Libdecor crashes on attempts to unset the parent by passing null, which is allowed by the
+     * toplevel spec, so just use the raw xdg-toplevel instead (that's what libdecor does
+     * internally anyways).
+     */
 #ifdef HAVE_LIBDECOR_H
-    if (viddata->shell.libdecor) {
-        if (!modal_data->shell_surface.libdecor.frame) {
-            return SDL_SetError("Modal window was hidden");
-        }
-        if (!parent_data->shell_surface.libdecor.frame) {
-            return SDL_SetError("Parent window was hidden");
-        }
-        libdecor_frame_set_parent(modal_data->shell_surface.libdecor.frame,
-                                  parent_data->shell_surface.libdecor.frame);
+    if (modal_data->shell_surface_type == WAYLAND_SURFACE_LIBDECOR && modal_data->shell_surface.libdecor.frame) {
+        modal_toplevel = libdecor_frame_get_xdg_toplevel(modal_data->shell_surface.libdecor.frame);
     } else
 #endif
-        if (viddata->shell.xdg) {
-        if (modal_data->shell_surface.xdg.roleobj.toplevel == NULL) {
-            return SDL_SetError("Modal window was hidden");
+        if (modal_data->shell_surface_type == WAYLAND_SURFACE_XDG_TOPLEVEL && modal_data->shell_surface.xdg.roleobj.toplevel) {
+        modal_toplevel = modal_data->shell_surface.xdg.roleobj.toplevel;
+    }
+
+    if (parent_data) {
+#ifdef HAVE_LIBDECOR_H
+        if (parent_data->shell_surface_type == WAYLAND_SURFACE_LIBDECOR && parent_data->shell_surface.libdecor.frame) {
+            parent_toplevel = libdecor_frame_get_xdg_toplevel(parent_data->shell_surface.libdecor.frame);
+        } else
+#endif
+            if (parent_data->shell_surface_type == WAYLAND_SURFACE_XDG_TOPLEVEL && parent_data->shell_surface.xdg.roleobj.toplevel) {
+            parent_toplevel = parent_data->shell_surface.xdg.roleobj.toplevel;
         }
-        if (parent_data->shell_surface.xdg.roleobj.toplevel == NULL) {
-            return SDL_SetError("Parent window was hidden");
+    }
+
+    if (modal_toplevel) {
+        xdg_toplevel_set_parent(modal_toplevel, parent_toplevel);
+
+        if (viddata->xdg_wm_dialog_v1) {
+            if (parent_toplevel) {
+                if (!modal_data->xdg_dialog_v1) {
+                    modal_data->xdg_dialog_v1 = xdg_wm_dialog_v1_get_xdg_dialog(viddata->xdg_wm_dialog_v1, modal_toplevel);
+                }
+
+                xdg_dialog_v1_set_modal(modal_data->xdg_dialog_v1);
+            } else if (modal_data->xdg_dialog_v1) {
+                xdg_dialog_v1_unset_modal(modal_data->xdg_dialog_v1);
+            }
         }
-        xdg_toplevel_set_parent(modal_data->shell_surface.xdg.roleobj.toplevel,
-                                parent_data->shell_surface.xdg.roleobj.toplevel);
-    } else {
-        return SDL_Unsupported();
     }
 
     return 0;
@@ -1653,6 +1677,10 @@ void Wayland_ShowWindow(SDL_VideoDevice *_this, SDL_Window *window)
     }
 
     /* Restore state that was set prior to this call */
+    if (window->flags & SDL_WINDOW_MODAL) {
+        Wayland_SetWindowModalFor(_this, window, window->parent);
+    }
+
     Wayland_SetWindowTitle(_this, window);
 
     /* We have to wait until the surface gets a "configure" event, or use of
@@ -2590,6 +2618,10 @@ void Wayland_DestroyWindow(SDL_VideoDevice *_this, SDL_Window *window)
             wp_fractional_scale_v1_destroy(wind->fractional_scale);
         }
 
+        if (wind->xdg_dialog_v1) {
+            xdg_dialog_v1_destroy(wind->xdg_dialog_v1);
+        }
+
         SDL_free(wind->outputs);
         SDL_free(wind->app_id);
 
diff --git a/src/video/wayland/SDL_waylandwindow.h b/src/video/wayland/SDL_waylandwindow.h
index 23f8806688918..24245cff55660 100644
--- a/src/video/wayland/SDL_waylandwindow.h
+++ b/src/video/wayland/SDL_waylandwindow.h
@@ -96,6 +96,7 @@ struct SDL_WindowData
     struct wp_viewport *viewport;
     struct wp_fractional_scale_v1 *fractional_scale;
     struct zxdg_exported_v2 *exported;
+    struct xdg_dialog_v1 *xdg_dialog_v1;
 
     SDL_AtomicInt swap_interval_ready;
 
@@ -172,6 +173,7 @@ struct SDL_WindowData
     SDL_bool fullscreen_was_positioned;
     SDL_bool show_hide_sync_required;
     SDL_bool scale_to_display;
+    SDL_bool modal_reparenting_required;
 
     SDL_HitTestResult hit_test_result;
 
diff --git a/src/video/windows/SDL_windowsvideo.c b/src/video/windows/SDL_windowsvideo.c
index 32b42ce725179..5dc929d872cd2 100644
--- a/src/video/windows/SDL_windowsvideo.c
+++ b/src/video/windows/SDL_windowsvideo.c
@@ -202,6 +202,7 @@ static SDL_VideoDevice *WIN_CreateDevice(void)
     device->SetWindowResizable = WIN_SetWindowResizable;
     device->SetWindowAlwaysOnTop = WIN_SetWindowAlwaysOnTop;
     device->SetWindowFullscreen = WIN_SetWindowFullscreen;
+    device->SetWindowModalFor = WIN_SetWindowModalFor;
 #if !defined(SDL_PLATFORM_XBOXONE) && !defined(SDL_PLATFORM_XBOXSERIES)
     device->GetWindowICCProfile = WIN_GetWindowICCProfile;
     device->SetWindowMouseRect = WIN_SetWindowMouseRect;
diff --git a/src/video/windows/SDL_windowswindow.c b/src/video/windows/SDL_windowswindow.c
index b2dd17b187498..605a4e7bd759b 100644
--- a/src/video/windows/SDL_windowswindow.c
+++ b/src/video/windows/SDL_windowswindow.c
@@ -984,6 +984,10 @@ void WIN_ShowWindow(SDL_VideoDevice *_this, SDL_Window *window)
         WIN_SetWindowPosition(_this, window);
     }
 
+    if (window->flags & SDL_WINDOW_MODAL) {
+        EnableWindow(window->parent->driverdata->hwnd, FALSE);
+    }
+
     hwnd = window->driverdata->hwnd;
     style = GetWindowLong(hwnd, GWL_EXSTYLE);
     if (style & WS_EX_NOACTIVATE) {
@@ -1006,6 +1010,11 @@ void WIN_ShowWindow(SDL_VideoDevice *_this, SDL_Window *window)
 void WIN_HideWindow(SDL_VideoDevice *_this, SDL_Window *window)
 {
     HWND hwnd = window->driverdata->hwnd;
+
+    if (window->flags & SDL_WINDOW_MODAL) {
+        EnableWindow(window->parent->driverdata->hwnd, TRUE);
+    }
+
     ShowWindow(hwnd, SW_HIDE);
 
     /* Transfer keyboard focus back to the parent */
@@ -1720,4 +1729,39 @@ void WIN_UpdateDarkModeForHWND(HWND hwnd)
     }
 }
 
+int WIN_SetWindowModalFor(SDL_VideoDevice *_this, SDL_Window *modal_window, SDL_Window *parent_window)
+{
+    SDL_WindowData *modal_data = modal_window->driverdata;
+    const LONG_PTR parent_hwnd = (LONG_PTR)(parent_window ? parent_window->driverdata->hwnd : NULL);
+    const LONG_PTR old_ptr = GetWindowLongPtr(modal_data->hwnd, GWLP_HWNDPARENT);
+    const DWORD style = GetWindowLong(modal_data->hwnd, GWL_STYLE);
+
+    if (old_ptr == parent_hwnd) {
+        return 0;
+    }
+
+    /* Reenable the old parent window. */
+    if (old_ptr) {
+        EnableWindow((HWND)old_ptr, TRUE);
+    }
+
+    if (!(style & WS_CHILD)) {
+        /* Despite the name, this changes the *owner* of a toplevel window, not
+         * the parent of a child window.
+         *
+         * https://devblogs.microsoft.com/oldnewthing/20100315-00/?p=14613
+         */
+        SetWindowLongPtr(modal_data->hwnd, GWLP_HWNDPARENT, parent_hwnd);
+    } else {
+        SetParent(modal_data->hwnd, (HWND)parent_hwnd);
+    }
+
+    /* Disable the new parent window if the modal window is visible. */
+    if (!(modal_window->flags & SDL_WINDOW_HIDDEN) && parent_hwnd) {
+        EnableWindow((HWND)parent_hwnd, FALSE);
+    }
+
+    return 0;
+}
+
 #endif /* SDL_VIDEO_DRIVER_WINDOWS */
diff --git a/src/video/windows/SDL_windowswindow.h b/src/video/windows/SDL_windowswindow.h
index 760c1a7a93c33..65f87daf4e255 100644
--- a/src/video/windows/SDL_windowswindow.h
+++ b/src/video/windows/

(Patch may be truncated, please check the link at the top of this post.)