SDL: cocoa: Set appropriate flags on fullscreen spaces windows

SDL Commits
1 
From e66896406d6cf44d1770e6e11d5275c7c635dfa1 Mon Sep 17 00:00:00 2001
From: Frank Praznik <[EMAIL REDACTED]>
Date: Thu, 21 Dec 2023 11:47:16 -0500
Subject: [PATCH] cocoa: Set appropriate flags on fullscreen spaces windows

Don't mark fullscreen windows as having a title bar if the window is borderless, or it can end up in a weird, pseudo-decorated state when leaving fullscreen if the borders were toggled back on while the window was fullscreen.

Setting the window styling when about to leave fullscreen caused issues with this as well and is no longer needed, as it is ensured that the window's resizable state won't change while in a fullscreen space, or in a transition period.
---
 src/video/cocoa/SDL_cocoawindow.m | 36 +++++++++++++++++++------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/src/video/cocoa/SDL_cocoawindow.m b/src/video/cocoa/SDL_cocoawindow.m
index 75df071db839..79f2c2843a85 100644
--- a/src/video/cocoa/SDL_cocoawindow.m
+++ b/src/video/cocoa/SDL_cocoawindow.m
@@ -1140,8 +1140,19 @@ - (void)windowDidChangeScreen:(NSNotification *)aNotification
 - (void)windowWillEnterFullScreen:(NSNotification *)aNotification
 {
     SDL_Window *window = _data.window;
+    NSUInteger flags = NSWindowStyleMaskClosable | NSWindowStyleMaskMiniaturizable | NSWindowStyleMaskResizable;
 
-    SetWindowStyle(window, (NSWindowStyleMaskTitled | NSWindowStyleMaskClosable | NSWindowStyleMaskMiniaturizable | NSWindowStyleMaskResizable));
+    /* Don't set the titled flag on a fullscreen window if the windowed-mode window
+     * is borderless, or the window can wind up in a weird, pseudo-decorated state
+     * when leaving fullscreen if the border flag was toggled on.
+     */
+    if (!(window->flags & SDL_WINDOW_BORDERLESS)) {
+        flags |= NSWindowStyleMaskTitled;
+    } else {
+        flags |= NSWindowStyleMaskBorderless;
+    }
+
+    SetWindowStyle(window, flags);
 
     _data.was_zoomed = !!(window->flags & SDL_WINDOW_MAXIMIZED);
 
@@ -1205,27 +1216,24 @@ - (void)windowWillExitFullScreen:(NSNotification *)aNotification
 
     isFullscreenSpace = NO;
     inFullscreenTransition = YES;
-
-    /* As of macOS 10.11, the window seems to need to be resizable when exiting
-       a Space, in order for it to resize back to its windowed-mode size.
-       As of macOS 10.15, the window decorations can go missing sometimes after
-       certain fullscreen-desktop->exlusive-fullscreen->windowed mode flows
-       sometimes. Making sure the style mask always uses the windowed mode style
-       when returning to windowed mode from a space (instead of using a pending
-       fullscreen mode style mask) seems to work around that issue.
-     */
-    SetWindowStyle(window, GetWindowWindowedStyle(window) | NSWindowStyleMaskResizable);
 }
 
 - (void)windowDidFailToExitFullScreen:(NSNotification *)aNotification
 {
     SDL_Window *window = _data.window;
+    NSUInteger flags = NSWindowStyleMaskClosable | NSWindowStyleMaskMiniaturizable | NSWindowStyleMaskResizable;
 
     if (window->is_destroying) {
         return;
     }
 
-    SetWindowStyle(window, (NSWindowStyleMaskTitled | NSWindowStyleMaskClosable | NSWindowStyleMaskMiniaturizable | NSWindowStyleMaskResizable));
+    if (!(window->flags & SDL_WINDOW_BORDERLESS)) {
+        flags |= NSWindowStyleMaskTitled;
+    } else {
+        flags |= NSWindowStyleMaskBorderless;
+    }
+    
+    SetWindowStyle(window, flags);
 
     isFullscreenSpace = YES;
     inFullscreenTransition = NO;
@@ -2456,7 +2464,7 @@ void Cocoa_SetWindowBordered(SDL_VideoDevice *_this, SDL_Window *window, SDL_boo
 void Cocoa_SetWindowResizable(SDL_VideoDevice *_this, SDL_Window *window, SDL_bool resizable)
 {
     @autoreleasepool {
-        /* Don't set this if we're in a space!
+        /* Don't set this if we're in or transitioning to/from a space!
          * The window will get permanently stuck if resizable is false.
          * -flibit
          */
@@ -2464,7 +2472,7 @@ void Cocoa_SetWindowResizable(SDL_VideoDevice *_this, SDL_Window *window, SDL_bo
         Cocoa_WindowListener *listener = data.listener;
         NSWindow *nswindow = data.nswindow;
         SDL_CocoaVideoData *videodata = data.videodata;
-        if (![listener isInFullscreenSpace]) {
+        if (![listener isInFullscreenSpace] && ![listener isInFullscreenSpaceTransition]) {
             SetWindowStyle(window, GetWindowStyle(window));
         }
         if (videodata.allow_spaces) {