sdl12-compat: Merge branch 'sulix-x11symquirkfix'

From 0fb5c56eaa5c74044c0830909627935718dd02f2 Mon Sep 17 00:00:00 2001
From: David Gow <[EMAIL REDACTED]>
Date: Mon, 17 Oct 2022 17:35:14 +0800
Subject: [PATCH 1/2] quirks: Fixes for symbol-based quirks to force X11

Make some changes to how the automatic quirks for apps which link
against gl[x]ew and Cg functions work. In particular:
- They can be overridden by a user-set SDL_VIDEODRIVER env variable.
- They only print warnings if debug logging
  (SDL12COMPAT_DEBUG_LOGGING=1) is enabled, to match the other quirks.
- The messages are no-longer wrapped.

Fixes: 2a96bb35d756 ("quirks: Force X11 on Linux if certain symbols are in the process.")
---
 src/SDL12_compat.c | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/SDL12_compat.c b/src/SDL12_compat.c
index cb97408e5..2451e7a5d 100644
--- a/src/SDL12_compat.c
+++ b/src/SDL12_compat.c
@@ -1307,6 +1307,7 @@ static void
 SDL12Compat_ApplyQuirks(void)
 {
     const char *exe_name = SDL12Compat_GetExeName();
+    const char *videodriver_env = SDL20_getenv("SDL_VIDEODRIVER");
     int i;
 
     if (*exe_name == '\0') {
@@ -1329,6 +1330,7 @@ SDL12Compat_ApplyQuirks(void)
     }
 
     #ifdef __linux__
+    if (!videodriver_env || SDL20_strcmp(videodriver_env, "x11"))
     {
         void *global_symbols = dlopen(NULL, RTLD_LOCAL|RTLD_NOW);
         SDL_bool force_x11 = SDL_FALSE;
@@ -1344,10 +1346,14 @@ SDL12Compat_ApplyQuirks(void)
         dlclose(global_symbols);
 
         if (force_x11) {
-            SDL20_Log("sdl12-compat: We are forcing this app to use X11, because it probably");
-            SDL20_Log("sdl12-compat: talks to an X server directly, outside of SDL. If possible,");
-            SDL20_Log("sdl12-compat: this app should be fixed, to be compatible with Wayland, etc.");
-            SDL20_setenv("SDL_VIDEODRIVER", "x11", 1);
+            if (!videodriver_env) {
+                if (WantDebugLogging) {
+                    SDL20_Log("Forcing this app to use X11, because it probably talks to an X server directly, outside of SDL. If possible, this app should be fixed, to be compatible with Wayland, etc.");
+                }
+                SDL20_setenv("SDL_VIDEODRIVER", "x11", 1);
+            } else if (videodriver_env && WantDebugLogging) {
+                SDL20_Log("This app looks like it requires X11, but the SDL_VIDEODRIVER environment variable is set to \"%s\". If you have issues, try setting SDL_VIDEODRIVER=x11", videodriver_env);
+            }
         }
     }
     #endif

From abb33cddffb7e7551acab724922225d88f13474f Mon Sep 17 00:00:00 2001
From: "Ryan C. Gordon" <icculus@icculus.org>
Date: Mon, 17 Oct 2022 08:44:52 -0400
Subject: [PATCH 2/2] quirks: Better X11 test.

- Do the test before loading SDL2, in case SDL2 is explicitly linked to Xlib.
- Reformatted code.
- Only dlclose() if dlopen() succeeded.
- Report that we're forcing X11 on a single line of text to stderr.

Reference Issue #249.
---
 src/SDL12_compat.c | 64 ++++++++++++++++++++++++----------------------
 1 file changed, 33 insertions(+), 31 deletions(-)

diff --git a/src/SDL12_compat.c b/src/SDL12_compat.c
index 2451e7a5d..9db296e49 100644
--- a/src/SDL12_compat.c
+++ b/src/SDL12_compat.c
@@ -1304,12 +1304,27 @@ SDL12Compat_GetHintFloat(const char *name, float default_value)
 }
 
 static void
-SDL12Compat_ApplyQuirks(void)
+SDL12Compat_ApplyQuirks(SDL_bool force_x11)
 {
     const char *exe_name = SDL12Compat_GetExeName();
-    const char *videodriver_env = SDL20_getenv("SDL_VIDEODRIVER");
     int i;
 
+    #ifdef __linux__
+    if (force_x11) {
+        const char *videodriver_env = SDL20_getenv("SDL_VIDEODRIVER");
+        if (videodriver_env && (SDL20_strcmp(videodriver_env, "x11") != 0)) {
+            if (WantDebugLogging) {
+                SDL20_Log("This app looks like it requires X11, but the SDL_VIDEODRIVER environment variable is set to \"%s\". If you have issues, try setting SDL_VIDEODRIVER=x11", videodriver_env);
+            }
+        } else {
+            if (WantDebugLogging) {
+                SDL20_Log("sdl12-compat: We are forcing this app to use X11, because it probably talks to an X server directly, outside of SDL. If possible, this app should be fixed, to be compatible with Wayland, etc.");
+            }
+            SDL20_setenv("SDL_VIDEODRIVER", "x11", 1);
+        }
+    }
+    #endif
+
     if (*exe_name == '\0') {
         return;
     }
@@ -1328,42 +1343,29 @@ SDL12Compat_ApplyQuirks(void)
             }
         }
     }
+}
 
-    #ifdef __linux__
-    if (!videodriver_env || SDL20_strcmp(videodriver_env, "x11"))
-    {
-        void *global_symbols = dlopen(NULL, RTLD_LOCAL|RTLD_NOW);
+static int
+LoadSDL20(void)
+{
+    int okay = 1;
+    if (!Loaded_SDL20) {
         SDL_bool force_x11 = SDL_FALSE;
 
+        #ifdef __linux__
+        void *global_symbols = dlopen(NULL, RTLD_LOCAL|RTLD_NOW);
+
         /* Use linked libraries to detect what quirks we are likely to need */
         if (global_symbols != NULL) {
-            /* GLEW (e.g. Frogatto, SLUDGE) */
-            if (dlsym(global_symbols, "glxewInit") != NULL) { force_x11 = SDL_TRUE; }
-            /* NVIDIA Cg (e.g. Awesomenauts, Braid) */
-            else if (dlsym(global_symbols, "cgGLEnableProgramProfiles") != NULL) { force_x11 = SDL_TRUE; }
-        }
-
-        dlclose(global_symbols);
-
-        if (force_x11) {
-            if (!videodriver_env) {
-                if (WantDebugLogging) {
-                    SDL20_Log("Forcing this app to use X11, because it probably talks to an X server directly, outside of SDL. If possible, this app should be fixed, to be compatible with Wayland, etc.");
-                }
-                SDL20_setenv("SDL_VIDEODRIVER", "x11", 1);
-            } else if (videodriver_env && WantDebugLogging) {
-                SDL20_Log("This app looks like it requires X11, but the SDL_VIDEODRIVER environment variable is set to \"%s\". If you have issues, try setting SDL_VIDEODRIVER=x11", videodriver_env);
+            if (dlsym(global_symbols, "glxewInit") != NULL) {  /* GLEW (e.g. Frogatto, SLUDGE) */
+                force_x11 = SDL_TRUE;
+            } else if (dlsym(global_symbols, "cgGLEnableProgramProfiles") != NULL) {  /* NVIDIA Cg (e.g. Awesomenauts, Braid) */
+                force_x11 = SDL_TRUE;
             }
+            dlclose(global_symbols);
         }
-    }
-    #endif
-}
+        #endif
 
-static int
-LoadSDL20(void)
-{
-    int okay = 1;
-    if (!Loaded_SDL20) {
         okay = LoadSDL20Library();
         if (!okay) {
             strcpy_fn(loaderror, "Failed loading SDL2 library.");
@@ -1386,7 +1388,7 @@ LoadSDL20(void)
                         SDL20_Log("sdl12-compat 1.2.%d, talking to SDL2 %d.%d.%d", SDL12_COMPAT_VERSION, v.major, v.minor, v.patch);
                         #endif
                     }
-                    SDL12Compat_ApplyQuirks();  /* Apply and maybe print a list of any enabled quirks. */
+                    SDL12Compat_ApplyQuirks(force_x11);  /* Apply and maybe print a list of any enabled quirks. */
                 }
             }
             if (!okay) {