SDL: Merge pull request #7238 from Kontrabant/wayland_fix_portrait

From b2d8a1a58cbf62a0fc080554e6dfd3f2a6ebbd99 Mon Sep 17 00:00:00 2001
From: Frank Praznik <[EMAIL REDACTED]>
Date: Sat, 4 Feb 2023 10:59:17 -0500
Subject: [PATCH] wayland: Transform mode values for native portrait displays

Portrait displays may have native, physical resolutions that are taller than wide. Reverse the mode dimensions when dealing with these displays as well as those rotated via software means.
---
 src/video/wayland/SDL_waylandvideo.c | 37 +++++++++++++---------------
 1 file changed, 17 insertions(+), 20 deletions(-)

diff --git a/src/video/wayland/SDL_waylandvideo.c b/src/video/wayland/SDL_waylandvideo.c
index 2246e20b83d4..39f0869c8144 100644
--- a/src/video/wayland/SDL_waylandvideo.c
+++ b/src/video/wayland/SDL_waylandvideo.c
@@ -361,7 +361,7 @@ static const struct zxdg_output_v1_listener xdg_output_listener = {
     xdg_output_handle_description,
 };
 
-static void AddEmulatedModes(SDL_DisplayData *dispdata, SDL_bool rot_90)
+static void AddEmulatedModes(SDL_DisplayData *dispdata, int native_width, int native_height)
 {
     struct EmulatedMode
     {
@@ -414,8 +414,7 @@ static void AddEmulatedModes(SDL_DisplayData *dispdata, SDL_bool rot_90)
     int i;
     SDL_DisplayMode mode;
     SDL_VideoDisplay *dpy = dispdata->display ? SDL_GetVideoDisplay(dispdata->display) : &dispdata->placeholder;
-    const int native_width = dispdata->pixel_width;
-    const int native_height = dispdata->pixel_height;
+    const SDL_bool rot_90 = native_width < native_height; /* Reverse width/height for portrait displays. */
 
     for (i = 0; i < SDL_arraysize(mode_list); ++i) {
         SDL_zero(mode);
@@ -424,19 +423,18 @@ static void AddEmulatedModes(SDL_DisplayData *dispdata, SDL_bool rot_90)
         mode.refresh_rate = dpy->desktop_mode.refresh_rate;
         mode.driverdata = dpy->desktop_mode.driverdata;
 
-        /* Only add modes that are smaller than the native mode. */
-        if ((mode_list[i].w < native_width && mode_list[i].h < native_height) ||
-            (mode_list[i].w < native_width && mode_list[i].h == native_height) ||
-            (mode_list[i].w == native_width && mode_list[i].h < native_height)) {
-
-            if (rot_90) {
-                mode.pixel_w = mode_list[i].h;
-                mode.pixel_h = mode_list[i].w;
-            } else {
-                mode.pixel_w = mode_list[i].w;
-                mode.pixel_h = mode_list[i].h;
-            }
+        if (rot_90) {
+            mode.pixel_w = mode_list[i].h;
+            mode.pixel_h = mode_list[i].w;
+        } else {
+            mode.pixel_w = mode_list[i].w;
+            mode.pixel_h = mode_list[i].h;
+        }
 
+        /* Only add modes that are smaller than the native mode. */
+        if ((mode.pixel_w < native_width && mode.pixel_h < native_height) ||
+            (mode.pixel_w < native_width && mode.pixel_h == native_height) ||
+            (mode.pixel_w == native_width && mode.pixel_h < native_height)) {
             SDL_AddFullscreenDisplayMode(dpy, &mode);
         }
     }
@@ -614,11 +612,11 @@ static void display_handle_done(void *data,
     /* Set the desktop display mode. */
     SDL_SetDesktopDisplayMode(dpy, &desktop_mode);
 
-    /* ...expose the unscaled, native resolution if the scale is 1.0 or viewports are available... */
+    /* Expose the unscaled, native resolution if the scale is 1.0 or viewports are available... */
     if (driverdata->scale_factor == 1.0f || video->viewporter != NULL) {
         SDL_AddFullscreenDisplayMode(dpy, &native_mode);
     } else {
-        /* ...if not, expose the integer scaled variants of the desktop resolution down to 1. */
+        /* ...otherwise expose the integer scaled variants of the desktop resolution down to 1. */
         int i;
 
         desktop_mode.pixel_w = 0;
@@ -634,9 +632,8 @@ static void display_handle_done(void *data,
 
     /* Add emulated modes if wp_viewporter is supported and mode emulation is enabled. */
     if (video->viewporter && mode_emulation_enabled) {
-        const SDL_bool rot_90 = (driverdata->transform & WL_OUTPUT_TRANSFORM_90) ||
-                                (driverdata->screen_width < driverdata->screen_height);
-        AddEmulatedModes(driverdata, rot_90);
+        /* The transformed display pixel width/height must be used here. */
+        AddEmulatedModes(driverdata, native_mode.pixel_w, native_mode.pixel_h);
     }
 
     /* Calculate the display DPI */