SDL: Fix dialogs on Windows

From 2b7af6636f50af83b65afb5636c04b690f895ca7 Mon Sep 17 00:00:00 2001
From: Semphris <[EMAIL REDACTED]>
Date: Sat, 27 Apr 2024 13:59:57 -0400
Subject: [PATCH] Fix dialogs on Windows

This fixes a bunch of issues with dialogs on Windows.

- Removed lpstrFileTitle assignation, which overwrote the buffer
- Increased the memory size available for long file selections
- Removed seemingly unused `default_folder` in win_Args struct
- Properly handle the case where only one file is selected in multiselect mode
- Properly handle the initial folder, which would fail in specific conditions

The details for the last entry are explained in a comment in the code.
---
 src/dialog/windows/SDL_windowsdialog.c | 91 ++++++++++++++++++++++----
 1 file changed, 77 insertions(+), 14 deletions(-)

diff --git a/src/dialog/windows/SDL_windowsdialog.c b/src/dialog/windows/SDL_windowsdialog.c
index edb4cbcf67121..2143f9b29d143 100644
--- a/src/dialog/windows/SDL_windowsdialog.c
+++ b/src/dialog/windows/SDL_windowsdialog.c
@@ -26,12 +26,14 @@
 #include "../../core/windows/SDL_windows.h"
 #include "../../thread/SDL_systhread.h"
 
+/* If this number is too small, selecting too many files will give an error */
+#define SELECTLIST_SIZE 65536
+
 typedef struct
 {
     int is_save;
     const SDL_DialogFileFilter *filters;
     const char* default_file;
-    const char* default_folder;
     SDL_Window* parent;
     DWORD flags;
     SDL_DialogFileCallback callback;
@@ -68,7 +70,6 @@ void windows_ShowFileDialog(void *ptr)
     int is_save = args->is_save;
     const SDL_DialogFileFilter *filters = args->filters;
     const char* default_file = args->default_file;
-    const char* default_folder = args->default_folder;
     SDL_Window* parent = args->parent;
     DWORD flags = args->flags;
     SDL_DialogFileCallback callback = args->callback;
@@ -109,18 +110,48 @@ void windows_ShowFileDialog(void *ptr)
         window = (HWND) SDL_GetProperty(SDL_GetWindowProperties(parent), SDL_PROP_WINDOW_WIN32_HWND_POINTER, NULL);
     }
 
-    wchar_t filebuffer[MAX_PATH] = L"";
-    wchar_t initfolder[MAX_PATH] = L"";
+    wchar_t *filebuffer; /* lpstrFile */
+    wchar_t initfolder[MAX_PATH] = L""; /* lpstrInitialDir */
+
+    /* If SELECTLIST_SIZE is too large, putting filebuffer on the stack might
+       cause an overflow */
+    filebuffer = (wchar_t *) SDL_malloc(SELECTLIST_SIZE * sizeof(wchar_t));
 
     /* Necessary for the return code below */
-    SDL_memset(filebuffer, 0, MAX_PATH * sizeof(wchar_t));
+    SDL_memset(filebuffer, 0, SELECTLIST_SIZE * sizeof(wchar_t));
 
     if (default_file) {
-        MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, default_file, -1, filebuffer, MAX_PATH);
-    }
+        /* On Windows 10, 11 and possibly others, lpstrFile can be initialized
+           with a path and the dialog will start at that location, but *only if
+           the path contains a filename*. If it ends with a folder (directory
+           separator), it fails with 0x3002 (12290) FNERR_INVALIDFILENAME. For
+           that specific case, lpstrInitialDir must be used instead, but just
+           for that case, because lpstrInitialDir doesn't support file names.
+
+           On top of that, lpstrInitialDir hides a special algorithm that
+           decides which folder to actually use as starting point, which may or
+           may not be the one provided, or some other unrelated folder. Also,
+           the algorithm changes between platforms. Assuming the documentation
+           is correct, the algorithm is there under 'lpstrInitialDir':
+
+           https://learn.microsoft.com/en-us/windows/win32/api/commdlg/ns-commdlg-openfilenamew
+
+           Finally, lpstrFile does not support forward slashes. lpstrInitialDir
+           does, though. */
+
+        char last_c = default_file[SDL_strlen(default_file) - 1];
+
+        if (last_c == '\\' || last_c == '/') {
+            MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, default_file, -1, initfolder, MAX_PATH);
+        } else {
+            MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, default_file, -1, filebuffer, MAX_PATH);
 
-    if (default_folder) {
-        MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, default_folder, -1, filebuffer, MAX_PATH);
+            for (int i = 0; i < SELECTLIST_SIZE; i++) {
+                if (filebuffer[i] == L'/') {
+                    filebuffer[i] = L'\\';
+                }
+            }
+        }
     }
 
     /* '\x01' is used in place of a null byte */
@@ -129,6 +160,7 @@ void windows_ShowFileDialog(void *ptr)
 
     if (!filterlist) {
         callback(userdata, NULL, -1);
+        SDL_free(filebuffer);
         return;
     }
 
@@ -147,6 +179,7 @@ void windows_ShowFileDialog(void *ptr)
         SDL_OutOfMemory();
         SDL_free(filterlist);
         callback(userdata, NULL, -1);
+        SDL_free(filebuffer);
         return;
     }
 
@@ -163,9 +196,8 @@ void windows_ShowFileDialog(void *ptr)
     dialog.nMaxCustFilter = 0;
     dialog.nFilterIndex = 0;
     dialog.lpstrFile = filebuffer;
-    dialog.nMaxFile = MAX_PATH;
-    dialog.lpstrFileTitle = *filebuffer ? filebuffer : NULL;
-    dialog.nMaxFileTitle = MAX_PATH;
+    dialog.nMaxFile = SELECTLIST_SIZE;
+    dialog.lpstrFileTitle = NULL;
     dialog.lpstrInitialDir = *initfolder ? initfolder : NULL;
     dialog.lpstrTitle = NULL;
     dialog.Flags = flags | OFN_EXPLORER | OFN_HIDEREADONLY | OFN_NOCHANGEDIR;
@@ -207,6 +239,7 @@ void windows_ShowFileDialog(void *ptr)
             if (!chosen_files_list) {
                 SDL_OutOfMemory();
                 callback(userdata, NULL, -1);
+                SDL_free(filebuffer);
                 return;
             }
 
@@ -216,6 +249,7 @@ void windows_ShowFileDialog(void *ptr)
                 SDL_SetError("Path too long or invalid character in path");
                 SDL_free(chosen_files_list);
                 callback(userdata, NULL, -1);
+                SDL_free(filebuffer);
                 return;
             }
 
@@ -238,6 +272,7 @@ void windows_ShowFileDialog(void *ptr)
 
                     SDL_free(chosen_files_list);
                     callback(userdata, NULL, -1);
+                    SDL_free(filebuffer);
                     return;
                 }
 
@@ -255,6 +290,7 @@ void windows_ShowFileDialog(void *ptr)
 
                     SDL_free(chosen_files_list);
                     callback(userdata, NULL, -1);
+                    SDL_free(filebuffer);
                     return;
                 }
 
@@ -271,6 +307,33 @@ void windows_ShowFileDialog(void *ptr)
 
                     SDL_free(chosen_files_list);
                     callback(userdata, NULL, -1);
+                    SDL_free(filebuffer);
+                    return;
+                }
+            }
+
+            /* If the user chose only one file, it's all just one string */
+            if (nfiles == 0) {
+                nfiles++;
+                char **new_cfl = (char **) SDL_realloc(chosen_files_list, sizeof(char*) * (nfiles + 1));
+
+                if (!new_cfl) {
+                    SDL_OutOfMemory();
+                    SDL_free(chosen_files_list);
+                    callback(userdata, NULL, -1);
+                    SDL_free(filebuffer);
+                    return;
+                }
+
+                chosen_files_list = new_cfl;
+                chosen_files_list[nfiles] = NULL;
+                chosen_files_list[nfiles - 1] = SDL_strdup(chosen_folder);
+
+                if (!chosen_files_list[nfiles - 1]) {
+                    SDL_OutOfMemory();
+                    SDL_free(chosen_files_list);
+                    callback(userdata, NULL, -1);
+                    SDL_free(filebuffer);
                     return;
                 }
             }
@@ -298,6 +361,8 @@ void windows_ShowFileDialog(void *ptr)
             callback(userdata, NULL, -1);
         }
     }
+
+    SDL_free(filebuffer);
 }
 
 int windows_file_dialog_thread(void* ptr)
@@ -400,7 +465,6 @@ void SDL_ShowOpenFileDialog(SDL_DialogFileCallback callback, void* userdata, SDL
     args->is_save = 0;
     args->filters = filters;
     args->default_file = default_location;
-    args->default_folder = NULL;
     args->parent = window;
     args->flags = (allow_many == SDL_TRUE) ? OFN_ALLOWMULTISELECT : 0;
     args->callback = callback;
@@ -438,7 +502,6 @@ void SDL_ShowSaveFileDialog(SDL_DialogFileCallback callback, void* userdata, SDL
     args->is_save = 1;
     args->filters = filters;
     args->default_file = default_location;
-    args->default_folder = NULL;
     args->parent = window;
     args->flags = 0;
     args->callback = callback;