SDL: Update the draggable state when the hit test callback changes

From 5e6ae85b63166290a5782a5cada3151bb35ee7e8 Mon Sep 17 00:00:00 2001
From: Sam Lantinga <[EMAIL REDACTED]>
Date: Tue, 13 Feb 2024 09:26:08 -0800
Subject: [PATCH] Update the draggable state when the hit test callback changes

Fixes https://github.com/libsdl-org/SDL/issues/7617
---
 src/video/SDL_video.c             |  4 +--
 src/video/cocoa/SDL_cocoawindow.h |  1 +
 src/video/cocoa/SDL_cocoawindow.m | 58 +++++++++++++++++++++++--------
 3 files changed, 46 insertions(+), 17 deletions(-)

diff --git a/src/video/SDL_video.c b/src/video/SDL_video.c
index 3804a95d96f6..89076d6aa6fd 100644
--- a/src/video/SDL_video.c
+++ b/src/video/SDL_video.c
@@ -5056,14 +5056,12 @@ int SDL_SetWindowHitTest(SDL_Window *window, SDL_HitTest callback, void *callbac
 
     if (!_this->SetWindowHitTest) {
         return SDL_Unsupported();
-    } else if (_this->SetWindowHitTest(window, callback != NULL) == -1) {
-        return -1;
     }
 
     window->hit_test = callback;
     window->hit_test_data = callback_data;
 
-    return 0;
+    return _this->SetWindowHitTest(window, callback != NULL);
 }
 
 int SDL_SetWindowShape(SDL_Window *window, SDL_Surface *shape)
diff --git a/src/video/cocoa/SDL_cocoawindow.h b/src/video/cocoa/SDL_cocoawindow.h
index 2831ad334f94..ea1f739d62ab 100644
--- a/src/video/cocoa/SDL_cocoawindow.h
+++ b/src/video/cocoa/SDL_cocoawindow.h
@@ -98,6 +98,7 @@ typedef enum
 - (NSApplicationPresentationOptions)window:(NSWindow *)window willUseFullScreenPresentationOptions:(NSApplicationPresentationOptions)proposedOptions;
 
 /* See if event is in a drag area, toggle on window dragging. */
+- (void)updateHitTest;
 - (BOOL)processHitTest:(NSEvent *)theEvent;
 
 /* Window event handling */
diff --git a/src/video/cocoa/SDL_cocoawindow.m b/src/video/cocoa/SDL_cocoawindow.m
index e8afbb74207b..df1e39221a7a 100644
--- a/src/video/cocoa/SDL_cocoawindow.m
+++ b/src/video/cocoa/SDL_cocoawindow.m
@@ -842,16 +842,16 @@ - (void)updateIgnoreMouseState:(NSEvent *)theEvent
 
     if (shape) {
         NSPoint point = [theEvent locationInWindow];
-		NSRect windowRect = [[_data.nswindow contentView] frame];
-		if (NSMouseInRect(point, windowRect, NO)) {
-			int x = (int)SDL_roundf((point.x / (window->w - 1)) * (shape->w - 1));
-			int y = (int)SDL_roundf(((window->h - point.y) / (window->h - 1)) * (shape->h - 1));
-			Uint8 a;
-
-			if (SDL_ReadSurfacePixel(shape, x, y, NULL, NULL, NULL, &a) < 0 || a == SDL_ALPHA_TRANSPARENT) {
-				ignoresMouseEvents = YES;
-			}
-		}
+        NSRect windowRect = [[_data.nswindow contentView] frame];
+        if (NSMouseInRect(point, windowRect, NO)) {
+            int x = (int)SDL_roundf((point.x / (window->w - 1)) * (shape->w - 1));
+            int y = (int)SDL_roundf(((window->h - point.y) / (window->h - 1)) * (shape->h - 1));
+            Uint8 a;
+
+            if (SDL_ReadSurfacePixel(shape, x, y, NULL, NULL, NULL, &a) < 0 || a == SDL_ALPHA_TRANSPARENT) {
+                ignoresMouseEvents = YES;
+            }
+        }
     }
     _data.nswindow.ignoresMouseEvents = ignoresMouseEvents;
 }
@@ -1404,14 +1404,41 @@ - (void)doCommandBySelector:(SEL)aSelector
     /*NSLog(@"doCommandBySelector: %@\n", NSStringFromSelector(aSelector));*/
 }
 
+- (void)updateHitTest
+{
+    SDL_Window *window = _data.window;
+    BOOL draggable = NO;
+
+    if (window->hit_test) {
+        float x, y;
+        SDL_Point point;
+
+        SDL_GetGlobalMouseState(&x, &y);
+        point.x = (int)SDL_roundf(x - window->x);
+        point.y = (int)SDL_roundf(y - window->y);
+        if (point.x >= 0 && point.x < window->w && point.y >= 0 && point.y < window->h) {
+            if (window->hit_test(window, &point, window->hit_test_data) == SDL_HITTEST_DRAGGABLE) {
+                draggable = YES;
+            }
+        }
+    }
+
+    if (isDragAreaRunning != draggable) {
+        isDragAreaRunning = draggable;
+        [_data.nswindow setMovableByWindowBackground:draggable];
+    }
+}
+
 - (BOOL)processHitTest:(NSEvent *)theEvent
 {
+    SDL_Window *window = _data.window;
+
     SDL_assert(isDragAreaRunning == [_data.nswindow isMovableByWindowBackground]);
 
-    if (_data.window->hit_test) { /* if no hit-test, skip this. */
+    if (window->hit_test) { /* if no hit-test, skip this. */
         const NSPoint location = [theEvent locationInWindow];
-        const SDL_Point point = { (int)location.x, _data.window->h - (((int)location.y) - 1) };
-        const SDL_HitTestResult rc = _data.window->hit_test(_data.window, &point, _data.window->hit_test_data);
+        const SDL_Point point = { (int)location.x, window->h - (((int)location.y) - 1) };
+        const SDL_HitTestResult rc = window->hit_test(window, &point, window->hit_test_data);
         if (rc == SDL_HITTEST_DRAGGABLE) {
             if (!isDragAreaRunning) {
                 isDragAreaRunning = YES;
@@ -2866,7 +2893,10 @@ take effect properly (e.g. setting the window size, etc.)
 
 int Cocoa_SetWindowHitTest(SDL_Window *window, SDL_bool enabled)
 {
-    return 0; /* just succeed, the real work is done elsewhere. */
+    SDL_CocoaWindowData *data = (__bridge SDL_CocoaWindowData *)window->driverdata;
+
+    [data.listener updateHitTest];
+    return 0;
 }
 
 void Cocoa_AcceptDragAndDrop(SDL_Window *window, SDL_bool accept)