SDL: Transparent window for Win32 + OpenGL (#8143)

From c7588e426128d1dbad92b1397b07f0dfb5ac50b3 Mon Sep 17 00:00:00 2001
From: Sam Huang <[EMAIL REDACTED]>
Date: Wed, 23 Aug 2023 07:42:59 -0700
Subject: [PATCH] Transparent window for Win32 + OpenGL (#8143)

* Transparent window for Win32 + OpenGL via DWM
---
 src/video/windows/SDL_windowswindow.c | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/src/video/windows/SDL_windowswindow.c b/src/video/windows/SDL_windowswindow.c
index 6218078e28df..c065166a6432 100644
--- a/src/video/windows/SDL_windowswindow.c
+++ b/src/video/windows/SDL_windowswindow.c
@@ -46,6 +46,16 @@
 #endif
 typedef HRESULT (WINAPI *DwmSetWindowAttribute_t)(HWND hwnd, DWORD dwAttribute, LPCVOID pvAttribute, DWORD cbAttribute);
 
+/* Transparent window support */
+typedef struct
+{
+    int left_width;
+    int right_width;
+    int top_height;
+    int bottom_height;
+} MARGINS;
+typedef HRESULT (WINAPI *DwmExtendFrameIntoClientArea_t)(HWND hwnd, const MARGINS *pMarInSet);
+
 /* Windows CE compatibility */
 #ifndef SWP_NOCOPYBITS
 #define SWP_NOCOPYBITS 0
@@ -587,6 +597,21 @@ int WIN_CreateWindow(SDL_VideoDevice *_this, SDL_Window *window)
         return 0;
     }
 
+    /* Only OpenGL has transparent framebuffer which is handled by above */
+    /* FIXME: Transparent window support for renderers other than OpenGL possible? */
+    if (window->flags & SDL_WINDOW_TRANSPARENT) {
+        void *handle = SDL_LoadObject("dwmapi.dll");
+        if (handle) {
+            DwmExtendFrameIntoClientArea_t DwmExtendFrameIntoClientAreaFunc = (DwmExtendFrameIntoClientArea_t)SDL_LoadFunction(handle, "DwmExtendFrameIntoClientArea");
+            if (DwmExtendFrameIntoClientAreaFunc) {
+                /* Negative margins create "sheet of glass" effect, thus transparent */
+                MARGINS margins = {-1, -1, -1, -1};
+                DwmExtendFrameIntoClientAreaFunc(hwnd, &margins);
+            }
+            SDL_UnloadObject(handle);
+        }
+    }
+
     /* The rest of this macro mess is for OpenGL or OpenGL ES windows */
 #ifdef SDL_VIDEO_OPENGL_ES2
     if ((_this->gl_config.profile_mask == SDL_GL_CONTEXT_PROFILE_ES ||