SDL: Handle interaction between auto capture and the SDL_CaptureMouse() API

From 86acb1a347fb232f0211864fe00af2d7590a3717 Mon Sep 17 00:00:00 2001
From: Sam Lantinga <[EMAIL REDACTED]>
Date: Tue, 5 Apr 2022 15:05:07 -0700
Subject: [PATCH] Handle interaction between auto capture and the
 SDL_CaptureMouse() API

Fixes https://github.com/libsdl-org/SDL/issues/5457
---
 src/events/SDL_keyboard.c |  1 +
 src/events/SDL_mouse.c    | 76 ++++++++++++++++++++++++---------------
 src/events/SDL_mouse_c.h  |  5 +++
 3 files changed, 53 insertions(+), 29 deletions(-)

diff --git a/src/events/SDL_keyboard.c b/src/events/SDL_keyboard.c
index cd150fab699..14e79ca49cd 100644
--- a/src/events/SDL_keyboard.c
+++ b/src/events/SDL_keyboard.c
@@ -638,6 +638,7 @@ SDL_SetKeyboardFocus(SDL_Window * window)
         /* old window must lose an existing mouse capture. */
         if (keyboard->focus->flags & SDL_WINDOW_MOUSE_CAPTURE) {
             SDL_CaptureMouse(SDL_FALSE);  /* drop the capture. */
+            SDL_UpdateMouseCapture(SDL_TRUE);
             SDL_assert(!(keyboard->focus->flags & SDL_WINDOW_MOUSE_CAPTURE));
         }
 
diff --git a/src/events/SDL_mouse.c b/src/events/SDL_mouse.c
index 092c95f40e4..f95c5b71c82 100644
--- a/src/events/SDL_mouse.c
+++ b/src/events/SDL_mouse.c
@@ -156,12 +156,8 @@ SDL_MouseAutoCaptureChanged(void *userdata, const char *name, const char *oldVal
     SDL_bool auto_capture = SDL_GetStringBoolean(hint, SDL_TRUE);
 
     if (auto_capture != mouse->auto_capture) {
-        /* Turn off mouse capture if it's currently active because of button presses */
-        if (!auto_capture && SDL_GetMouseState(NULL, NULL) != 0) {
-            SDL_CaptureMouse(SDL_FALSE);
-        }
-
         mouse->auto_capture = auto_capture;
+        SDL_UpdateMouseCapture(SDL_FALSE);
     }
 }
 
@@ -668,10 +664,7 @@ SDL_PrivateSendMouseButton(SDL_Window * window, SDL_MouseID mouseID, Uint8 state
 
     /* Automatically capture the mouse while buttons are pressed */
     if (mouse->auto_capture) {
-        SDL_bool has_buttons_pressed = (SDL_GetMouseState(NULL, NULL) ? SDL_TRUE : SDL_FALSE);
-        if (has_buttons_pressed != had_buttons_pressed) {
-            SDL_CaptureMouse(has_buttons_pressed);
-        }
+        SDL_UpdateMouseCapture(SDL_FALSE);
     }
 
     return posted;
@@ -768,6 +761,7 @@ SDL_MouseQuit(void)
 
     if (mouse->CaptureMouse) {
         SDL_CaptureMouse(SDL_FALSE);
+        SDL_UpdateMouseCapture(SDL_TRUE);
     }
     SDL_SetRelativeMouseMode(SDL_FALSE);
     SDL_ShowCursor(1);
@@ -972,6 +966,8 @@ SDL_SetRelativeMouseMode(SDL_bool enabled)
         if (!enabled) {
             SDL_WarpMouseInWindow(focusWindow, mouse->x, mouse->y);
         }
+
+        SDL_UpdateMouseCapture(SDL_FALSE);
     }
 
     if (!enabled) {
@@ -994,40 +990,62 @@ SDL_GetRelativeMouseMode()
 }
 
 int
-SDL_CaptureMouse(SDL_bool enabled)
+SDL_UpdateMouseCapture(SDL_bool force_release)
 {
     SDL_Mouse *mouse = SDL_GetMouse();
-    SDL_Window *focusWindow;
-    SDL_bool isCaptured;
+    SDL_Window *capture_window = NULL;
 
     if (!mouse->CaptureMouse) {
-        return SDL_Unsupported();
+        return 0;
     }
 
-    focusWindow = SDL_GetKeyboardFocus();
-
-    isCaptured = focusWindow && (focusWindow->flags & SDL_WINDOW_MOUSE_CAPTURE);
-    if (isCaptured == enabled) {
-        return 0;  /* already done! */
+    if (!force_release) {
+        if (mouse->capture_desired || (mouse->auto_capture && SDL_GetMouseState(NULL, NULL) != 0)) {
+            if (!mouse->relative_mode) {
+                capture_window = SDL_GetKeyboardFocus();
+            }
+        }
     }
 
-    if (enabled) {
-        if (!focusWindow) {
-            return SDL_SetError("No window has focus");
-        } else if (mouse->CaptureMouse(focusWindow) == -1) {
-            return -1;  /* CaptureMouse() should call SetError */
+    if (capture_window != mouse->capture_window) {
+        if (mouse->capture_window) {
+            mouse->CaptureMouse(NULL);
+            mouse->capture_window->flags &= ~SDL_WINDOW_MOUSE_CAPTURE;
+            mouse->capture_window = NULL;
         }
-        focusWindow->flags |= SDL_WINDOW_MOUSE_CAPTURE;
-    } else {
-        if (mouse->CaptureMouse(NULL) == -1) {
-            return -1;  /* CaptureMouse() should call SetError */
+
+        if (capture_window) {
+            if (mouse->CaptureMouse(capture_window) < 0) {
+                /* CaptureMouse() will have set an error */
+                return -1;
+            }
+            capture_window->flags |= SDL_WINDOW_MOUSE_CAPTURE;
         }
-        focusWindow->flags &= ~SDL_WINDOW_MOUSE_CAPTURE;
-    }
 
+        mouse->capture_window = capture_window;
+    }
     return 0;
 }
 
+int
+SDL_CaptureMouse(SDL_bool enabled)
+{
+    SDL_Mouse *mouse = SDL_GetMouse();
+    SDL_Window *focus_window;
+
+    if (!mouse->CaptureMouse) {
+        return SDL_Unsupported();
+    }
+
+    focus_window = SDL_GetKeyboardFocus();
+    if (enabled && !focus_window) {
+        return SDL_SetError("No window has focus");
+    }
+    mouse->capture_desired = enabled;
+
+    return SDL_UpdateMouseCapture(SDL_FALSE);
+}
+
 SDL_Cursor *
 SDL_CreateCursor(const Uint8 * data, const Uint8 * mask,
                  int w, int h, int hot_x, int hot_y)
diff --git a/src/events/SDL_mouse_c.h b/src/events/SDL_mouse_c.h
index e2298d981e0..f06934b6fcb 100644
--- a/src/events/SDL_mouse_c.h
+++ b/src/events/SDL_mouse_c.h
@@ -104,6 +104,8 @@ typedef struct
     Uint8 vita_touch_mouse_device;
 #endif
     SDL_bool auto_capture;
+    SDL_bool capture_desired;
+    SDL_Window *capture_window;
 
     /* Data for input source state */
     int num_sources;
@@ -135,6 +137,9 @@ extern void SDL_SetDefaultCursor(SDL_Cursor * cursor);
 /* Set the mouse focus window */
 extern void SDL_SetMouseFocus(SDL_Window * window);
 
+/* Update the mouse capture window */
+extern int SDL_UpdateMouseCapture(SDL_bool force_release);
+
 /* Send a mouse motion event */
 extern int SDL_SendMouseMotion(SDL_Window * window, SDL_MouseID mouseID, int relative, int x, int y);