sdl12-compat: quirks: Better X11 test. (abb33)

From abb33cddffb7e7551acab724922225d88f13474f Mon Sep 17 00:00:00 2001
From: "Ryan C. Gordon" <[EMAIL REDACTED]>
Date: Mon, 17 Oct 2022 08:44:52 -0400
Subject: [PATCH] quirks: Better X11 test.

- Do the test before loading SDL2, in case SDL2 is explicitly linked to Xlib.
- Reformatted code.
- Only dlclose() if dlopen() succeeded.
- Report that we're forcing X11 on a single line of text to stderr.

Reference Issue #249.
---
 src/SDL12_compat.c | 64 ++++++++++++++++++++++++----------------------
 1 file changed, 33 insertions(+), 31 deletions(-)

diff --git a/src/SDL12_compat.c b/src/SDL12_compat.c
index 2451e7a5d..9db296e49 100644
--- a/src/SDL12_compat.c
+++ b/src/SDL12_compat.c
@@ -1304,12 +1304,27 @@ SDL12Compat_GetHintFloat(const char *name, float default_value)
 }
 
 static void
-SDL12Compat_ApplyQuirks(void)
+SDL12Compat_ApplyQuirks(SDL_bool force_x11)
 {
     const char *exe_name = SDL12Compat_GetExeName();
-    const char *videodriver_env = SDL20_getenv("SDL_VIDEODRIVER");
     int i;
 
+    #ifdef __linux__
+    if (force_x11) {
+        const char *videodriver_env = SDL20_getenv("SDL_VIDEODRIVER");
+        if (videodriver_env && (SDL20_strcmp(videodriver_env, "x11") != 0)) {
+            if (WantDebugLogging) {
+                SDL20_Log("This app looks like it requires X11, but the SDL_VIDEODRIVER environment variable is set to \"%s\". If you have issues, try setting SDL_VIDEODRIVER=x11", videodriver_env);
+            }
+        } else {
+            if (WantDebugLogging) {
+                SDL20_Log("sdl12-compat: We are forcing this app to use X11, because it probably talks to an X server directly, outside of SDL. If possible, this app should be fixed, to be compatible with Wayland, etc.");
+            }
+            SDL20_setenv("SDL_VIDEODRIVER", "x11", 1);
+        }
+    }
+    #endif
+
     if (*exe_name == '\0') {
         return;
     }
@@ -1328,42 +1343,29 @@ SDL12Compat_ApplyQuirks(void)
             }
         }
     }
+}
 
-    #ifdef __linux__
-    if (!videodriver_env || SDL20_strcmp(videodriver_env, "x11"))
-    {
-        void *global_symbols = dlopen(NULL, RTLD_LOCAL|RTLD_NOW);
+static int
+LoadSDL20(void)
+{
+    int okay = 1;
+    if (!Loaded_SDL20) {
         SDL_bool force_x11 = SDL_FALSE;
 
+        #ifdef __linux__
+        void *global_symbols = dlopen(NULL, RTLD_LOCAL|RTLD_NOW);
+
         /* Use linked libraries to detect what quirks we are likely to need */
         if (global_symbols != NULL) {
-            /* GLEW (e.g. Frogatto, SLUDGE) */
-            if (dlsym(global_symbols, "glxewInit") != NULL) { force_x11 = SDL_TRUE; }
-            /* NVIDIA Cg (e.g. Awesomenauts, Braid) */
-            else if (dlsym(global_symbols, "cgGLEnableProgramProfiles") != NULL) { force_x11 = SDL_TRUE; }
-        }
-
-        dlclose(global_symbols);
-
-        if (force_x11) {
-            if (!videodriver_env) {
-                if (WantDebugLogging) {
-                    SDL20_Log("Forcing this app to use X11, because it probably talks to an X server directly, outside of SDL. If possible, this app should be fixed, to be compatible with Wayland, etc.");
-                }
-                SDL20_setenv("SDL_VIDEODRIVER", "x11", 1);
-            } else if (videodriver_env && WantDebugLogging) {
-                SDL20_Log("This app looks like it requires X11, but the SDL_VIDEODRIVER environment variable is set to \"%s\". If you have issues, try setting SDL_VIDEODRIVER=x11", videodriver_env);
+            if (dlsym(global_symbols, "glxewInit") != NULL) {  /* GLEW (e.g. Frogatto, SLUDGE) */
+                force_x11 = SDL_TRUE;
+            } else if (dlsym(global_symbols, "cgGLEnableProgramProfiles") != NULL) {  /* NVIDIA Cg (e.g. Awesomenauts, Braid) */
+                force_x11 = SDL_TRUE;
             }
+            dlclose(global_symbols);
         }
-    }
-    #endif
-}
+        #endif
 
-static int
-LoadSDL20(void)
-{
-    int okay = 1;
-    if (!Loaded_SDL20) {
         okay = LoadSDL20Library();
         if (!okay) {
             strcpy_fn(loaderror, "Failed loading SDL2 library.");
@@ -1386,7 +1388,7 @@ LoadSDL20(void)
                         SDL20_Log("sdl12-compat 1.2.%d, talking to SDL2 %d.%d.%d", SDL12_COMPAT_VERSION, v.major, v.minor, v.patch);
                         #endif
                     }
-                    SDL12Compat_ApplyQuirks();  /* Apply and maybe print a list of any enabled quirks. */
+                    SDL12Compat_ApplyQuirks(force_x11);  /* Apply and maybe print a list of any enabled quirks. */
                 }
             }
             if (!okay) {