SDL: wayland: Set/unset the opaque regions on surfaces when transparency is toggled

From c2b0c41c0af0b5fbc589dfc0469e0b0c0e4f564a Mon Sep 17 00:00:00 2001
From: Frank Praznik <[EMAIL REDACTED]>
Date: Tue, 4 Oct 2022 12:59:26 -0400
Subject: [PATCH] wayland: Set/unset the opaque regions on surfaces when
 transparency is toggled

Caches the SDL_HINT_VIDEO_EGL_ALLOW_TRANSPARENCY hint at init time and registers a callback, which is fired when the hint is changed during runtime and toggles the opaque region for existing surfaces.
---
 src/video/wayland/SDL_waylandvideo.c  |  2 ++
 src/video/wayland/SDL_waylandvideo.h  |  1 +
 src/video/wayland/SDL_waylandwindow.c | 51 +++++++++++++++++++++++----
 src/video/wayland/SDL_waylandwindow.h |  3 ++
 4 files changed, 50 insertions(+), 7 deletions(-)

diff --git a/src/video/wayland/SDL_waylandvideo.c b/src/video/wayland/SDL_waylandvideo.c
index b9b1ce25133b..2a54d28cb5e7 100644
--- a/src/video/wayland/SDL_waylandvideo.c
+++ b/src/video/wayland/SDL_waylandvideo.c
@@ -992,6 +992,7 @@ Wayland_VideoInit(_THIS)
     WAYLAND_wl_display_flush(data->display);
 
     Wayland_InitKeyboard(_this);
+    Wayland_InitWin(data);
 
     data->initializing = SDL_FALSE;
 
@@ -1033,6 +1034,7 @@ Wayland_VideoQuit(_THIS)
     SDL_VideoData *data = _this->driverdata;
     int i, j;
 
+    Wayland_QuitWin(data);
     Wayland_FiniMouse(data);
 
     for (i = 0; i < _this->num_displays; ++i) {
diff --git a/src/video/wayland/SDL_waylandvideo.h b/src/video/wayland/SDL_waylandvideo.h
index 3e7b363c53fa..e1825cbae53c 100644
--- a/src/video/wayland/SDL_waylandvideo.h
+++ b/src/video/wayland/SDL_waylandvideo.h
@@ -95,6 +95,7 @@ typedef struct {
     char *classname;
 
     int relative_mouse_mode;
+    SDL_bool egl_transparency_enabled;
 } SDL_VideoData;
 
 struct SDL_WaylandOutputData {
diff --git a/src/video/wayland/SDL_waylandwindow.c b/src/video/wayland/SDL_waylandwindow.c
index a4e9f591e121..f949c20bc4cf 100644
--- a/src/video/wayland/SDL_waylandwindow.c
+++ b/src/video/wayland/SDL_waylandwindow.c
@@ -32,6 +32,7 @@
 #include "SDL_waylandvideo.h"
 #include "SDL_waylandtouch.h"
 #include "SDL_hints.h"
+#include "../../SDL_hints_c.h"
 #include "SDL_events.h"
 
 #include "xdg-shell-client-protocol.h"
@@ -46,12 +47,6 @@
 
 #define FULLSCREEN_MASK (SDL_WINDOW_FULLSCREEN | SDL_WINDOW_FULLSCREEN_DESKTOP)
 
-SDL_FORCE_INLINE SDL_bool
-EGLTransparencyEnabled()
-{
-    return SDL_GetHintBoolean(SDL_HINT_VIDEO_EGL_ALLOW_TRANSPARENCY, SDL_FALSE);
-}
-
 SDL_FORCE_INLINE SDL_bool
 FloatEqual(float a, float b)
 {
@@ -296,7 +291,7 @@ ConfigureWindowGeometry(SDL_Window *window)
      * if the output size has changed.
      */
     if (window_size_changed) {
-        if (!EGLTransparencyEnabled()) {
+        if (!viddata->egl_transparency_enabled) {
             region = wl_compositor_create_region(viddata->compositor);
             wl_region_add(region, 0, 0,
                           data->window_width, data->window_height);
@@ -2310,6 +2305,48 @@ void Wayland_DestroyWindow(_THIS, SDL_Window *window)
     window->driverdata = NULL;
 }
 
+static void
+EGLTransparencyChangedCallback(void *userdata, const char *name, const char *oldValue, const char *newValue)
+{
+    const SDL_bool oldval = SDL_GetStringBoolean(oldValue, SDL_FALSE);
+    const SDL_bool newval = SDL_GetStringBoolean(newValue, SDL_FALSE);
+
+    if (oldval != newval) {
+        SDL_Window      *window;
+        SDL_VideoData   *viddata = (SDL_VideoData *) userdata;
+        SDL_VideoDevice *dev     = SDL_GetVideoDevice();
+
+        viddata->egl_transparency_enabled = newval;
+
+        /* Iterate over all windows and update the surface opaque regions */
+        for (window = dev->windows; window != NULL; window = window->next) {
+            SDL_WindowData *wind = (SDL_WindowData *) window->driverdata;
+            
+            if (!newval) {
+                struct wl_region *region = wl_compositor_create_region(wind->waylandData->compositor);
+                wl_region_add(region, 0, 0, wind->window_width, wind->window_height);
+                wl_surface_set_opaque_region(wind->surface, region);
+                wl_region_destroy(region);
+            } else {
+                wl_surface_set_opaque_region(wind->surface, NULL);
+            }
+        }
+    }
+}
+
+void
+Wayland_InitWin(SDL_VideoData *data)
+{
+    data->egl_transparency_enabled = SDL_GetHintBoolean(SDL_HINT_VIDEO_EGL_ALLOW_TRANSPARENCY, SDL_FALSE);
+    SDL_AddHintCallback(SDL_HINT_VIDEO_EGL_ALLOW_TRANSPARENCY, EGLTransparencyChangedCallback, data);
+}
+
+void
+Wayland_QuitWin(SDL_VideoData *data)
+{
+    SDL_DelHintCallback(SDL_HINT_VIDEO_EGL_ALLOW_TRANSPARENCY, EGLTransparencyChangedCallback, data);
+}
+
 #endif /* SDL_VIDEO_DRIVER_WAYLAND */
 
 /* vi: set ts=4 sw=4 expandtab: */
diff --git a/src/video/wayland/SDL_waylandwindow.h b/src/video/wayland/SDL_waylandwindow.h
index a14ba13649e8..87d10eb435f4 100644
--- a/src/video/wayland/SDL_waylandwindow.h
+++ b/src/video/wayland/SDL_waylandwindow.h
@@ -142,6 +142,9 @@ Wayland_GetWindowWMInfo(_THIS, SDL_Window * window, SDL_SysWMinfo * info);
 extern int Wayland_SetWindowHitTest(SDL_Window *window, SDL_bool enabled);
 extern int Wayland_FlashWindow(_THIS, SDL_Window * window, SDL_FlashOperation operation);
 
+extern void Wayland_InitWin(SDL_VideoData *data);
+extern void Wayland_QuitWin(SDL_VideoData *data);
+
 #endif /* SDL_waylandwindow_h_ */
 
 /* vi: set ts=4 sw=4 expandtab: */