SDL: Update mouse transparency when the window shape changes

From f4d97bdb14a39c3d2151f122f57cf1f972c421a6 Mon Sep 17 00:00:00 2001
From: Sam Lantinga <[EMAIL REDACTED]>
Date: Mon, 12 Feb 2024 10:49:45 -0800
Subject: [PATCH] Update mouse transparency when the window shape changes

Fixes github.com/libsdl-org/SDL/issues/7617
---
 src/video/cocoa/SDL_cocoashape.h | 28 +++++++++++++++++
 src/video/cocoa/SDL_cocoashape.m | 54 ++++++++++++++++++++++++++++++++
 src/video/cocoa/SDL_cocoavideo.m |  2 ++
 src/video/x11/SDL_x11video.c     |  2 +-
 4 files changed, 85 insertions(+), 1 deletion(-)
 create mode 100644 src/video/cocoa/SDL_cocoashape.h
 create mode 100644 src/video/cocoa/SDL_cocoashape.m

diff --git a/src/video/cocoa/SDL_cocoashape.h b/src/video/cocoa/SDL_cocoashape.h
new file mode 100644
index 000000000000..b2d1d0cf2960
--- /dev/null
+++ b/src/video/cocoa/SDL_cocoashape.h
@@ -0,0 +1,28 @@
+/*
+  Simple DirectMedia Layer
+  Copyright (C) 1997-2024 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+*/
+#include "SDL_internal.h"
+
+#ifndef SDL_cocoashape_h_
+#define SDL_cocoashape_h_
+
+extern int Cocoa_UpdateWindowShape(SDL_VideoDevice *_this, SDL_Window *window, SDL_Surface *shape);
+
+#endif /* SDL_cocoashape_h_ */
diff --git a/src/video/cocoa/SDL_cocoashape.m b/src/video/cocoa/SDL_cocoashape.m
new file mode 100644
index 000000000000..633db6bd73dc
--- /dev/null
+++ b/src/video/cocoa/SDL_cocoashape.m
@@ -0,0 +1,54 @@
+/*
+  Simple DirectMedia Layer
+  Copyright (C) 1997-2024 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+*/
+#include "SDL_internal.h"
+
+#ifdef SDL_VIDEO_DRIVER_COCOA
+
+#include "SDL_cocoavideo.h"
+#include "SDL_cocoashape.h"
+
+
+int Cocoa_UpdateWindowShape(SDL_VideoDevice *_this, SDL_Window *window, SDL_Surface *shape)
+{
+    SDL_CocoaWindowData *data = (__bridge SDL_CocoaWindowData *)window->driverdata;
+    BOOL ignoresMouseEvents = NO;
+
+    if (shape) {
+        SDL_FPoint point;
+        SDL_GetGlobalMouseState(&point.x, &point.y);
+        point.x -= window->x;
+        point.y -= window->y;
+        if (point.x >= 0.0f && point.x < window->w &&
+            point.y >= 0.0f && point.y < window->h) {
+            int x = (int)SDL_roundf((point.x / (window->w - 1)) * (shape->w - 1));
+            int y = (int)SDL_roundf((point.y / (window->h - 1)) * (shape->h - 1));
+            Uint8 a;
+
+            if (SDL_ReadSurfacePixel(shape, x, y, NULL, NULL, NULL, &a) < 0 || a == SDL_ALPHA_TRANSPARENT) {
+                ignoresMouseEvents = YES;
+            }
+        }
+    }
+    data.nswindow.ignoresMouseEvents = ignoresMouseEvents;
+    return 0;
+}
+
+#endif /* SDL_VIDEO_DRIVER_COCOA */
diff --git a/src/video/cocoa/SDL_cocoavideo.m b/src/video/cocoa/SDL_cocoavideo.m
index 1f25d7434995..971892f11b79 100644
--- a/src/video/cocoa/SDL_cocoavideo.m
+++ b/src/video/cocoa/SDL_cocoavideo.m
@@ -31,6 +31,7 @@
 #include "SDL_cocoametalview.h"
 #include "SDL_cocoaopengles.h"
 #include "SDL_cocoamessagebox.h"
+#include "SDL_cocoashape.h"
 
 @implementation SDL_CocoaVideoData
 
@@ -115,6 +116,7 @@ static void Cocoa_DeleteDevice(SDL_VideoDevice *device)
         device->DestroyWindow = Cocoa_DestroyWindow;
         device->SetWindowHitTest = Cocoa_SetWindowHitTest;
         device->AcceptDragAndDrop = Cocoa_AcceptDragAndDrop;
+        device->UpdateWindowShape = Cocoa_UpdateWindowShape;
         device->FlashWindow = Cocoa_FlashWindow;
         device->SetWindowFocusable = Cocoa_SetWindowFocusable;
         device->SyncWindow = Cocoa_SyncWindow;
diff --git a/src/video/x11/SDL_x11video.c b/src/video/x11/SDL_x11video.c
index bd568461dde7..e62889f5b289 100644
--- a/src/video/x11/SDL_x11video.c
+++ b/src/video/x11/SDL_x11video.c
@@ -208,8 +208,8 @@ static SDL_VideoDevice *X11_CreateDevice(void)
     device->DestroyWindowFramebuffer = X11_DestroyWindowFramebuffer;
     device->SetWindowHitTest = X11_SetWindowHitTest;
     device->AcceptDragAndDrop = X11_AcceptDragAndDrop;
-    device->FlashWindow = X11_FlashWindow;
     device->UpdateWindowShape = X11_UpdateWindowShape;
+    device->FlashWindow = X11_FlashWindow;
     device->ShowWindowSystemMenu = X11_ShowWindowSystemMenu;
     device->SetWindowFocusable = X11_SetWindowFocusable;
     device->SyncWindow = X11_SyncWindow;