sdl12-compat: Don't set {R,G,B,A}masks on SDL2 paletted surfaces

From 96c74584d7b04007f8869e3f299b62d29c86abbb Mon Sep 17 00:00:00 2001
From: David Gow <[EMAIL REDACTED]>
Date: Tue, 22 Jun 2021 14:09:33 +0800
Subject: [PATCH] Don't set {R,G,B,A}masks on SDL2 paletted surfaces

SDL2 expects 8-bit surfaces to have correct {R,G,B,A}masks in their
format: either all 0 for paletted surfaces, or the correct values for
SDL_PIXELFORMAT_RGB332. Indeed, SDL_ConvertSurface() will crash if this
is not the case.

Some SDL 1.2 applications provide incorrect masks for paletted surfaces.
We originally zeroed these out in 46fdbf7c1d, but 3af751b91b actually
re-sets these in SetPalette12ForMasks().

Instead, do not set the masks on the SDL 2 surface in
SetPalette12ForMasks(), and force them to be 0 in PixelFormat12to20() if
there is a palette. The masks are still set on the SDL 1.2 surface's
format, and the palette generated by SetPalette12ForMasks() should in
theory make sure surfaces with 8-bit masks still work.

This fixes a crash on startup in circuslinux:
http://www.newbreedsoftware.com/circus-linux/
---
 src/SDL12_compat.c | 52 ++++++++++++++++++++++++++--------------------
 1 file changed, 29 insertions(+), 23 deletions(-)

diff --git a/src/SDL12_compat.c b/src/SDL12_compat.c
index ee352a3..e333678 100644
--- a/src/SDL12_compat.c
+++ b/src/SDL12_compat.c
@@ -1415,18 +1415,35 @@ PixelFormat12to20(SDL_PixelFormat *format20, SDL_Palette *palette20, const SDL12
     format20->format = SDL20_MasksToPixelFormatEnum(format12->BitsPerPixel, format12->Rmask, format12->Gmask, format12->Bmask, format12->Amask);
     format20->BitsPerPixel = format12->BitsPerPixel;
     format20->BytesPerPixel = format12->BytesPerPixel;
-    format20->Rmask = format12->Rmask;
-    format20->Gmask = format12->Gmask;
-    format20->Bmask = format12->Bmask;
-    format20->Amask = format12->Amask;
-    format20->Rloss = format12->Rloss;
-    format20->Gloss = format12->Gloss;
-    format20->Bloss = format12->Bloss;
-    format20->Aloss = format12->Aloss;
-    format20->Rshift = format12->Rshift;
-    format20->Gshift = format12->Gshift;
-    format20->Bshift = format12->Bshift;
-    format20->Ashift = format12->Ashift;
+
+    /* Paletted surfaces shouldn't have masks in SDL 2.0 */
+    if (format12->palette) {
+        format20->Rmask = 0;
+        format20->Gmask = 0;
+        format20->Bmask = 0;
+        format20->Amask = 0;
+        format20->Rloss = 8;
+        format20->Gloss = 8;
+        format20->Bloss = 8;
+        format20->Aloss = 8;
+        format20->Rshift = 0;
+        format20->Gshift = 0;
+        format20->Bshift = 0;
+        format20->Ashift = 0;
+    } else {
+        format20->Rmask = format12->Rmask;
+        format20->Gmask = format12->Gmask;
+        format20->Bmask = format12->Bmask;
+        format20->Amask = format12->Amask;
+        format20->Rloss = format12->Rloss;
+        format20->Gloss = format12->Gloss;
+        format20->Bloss = format12->Bloss;
+        format20->Aloss = format12->Aloss;
+        format20->Rshift = format12->Rshift;
+        format20->Gshift = format12->Gshift;
+        format20->Bshift = format12->Bshift;
+        format20->Ashift = format12->Ashift;
+    }
     format20->refcount = 1;
     format20->next = NULL;
     return format20;
@@ -3200,7 +3217,6 @@ static void
 SetPalette12ForMasks(SDL12_Surface *surface12, const Uint32 Rmask, const Uint32 Gmask, const Uint32 Bmask)
 {
     SDL12_PixelFormat *format12;
-    SDL_PixelFormat  * format20;
     SDL_Color *color;
     int i, ncolors;
 
@@ -3258,16 +3274,6 @@ SetPalette12ForMasks(SDL12_Surface *surface12, const Uint32 Rmask, const Uint32
             color->a = 255;
         }
 
-        format20 = surface12->surface20->format;
-        #define UPDATEFMT20(t) \
-            format20->t##mask = format12->t##mask; \
-            format20->t##loss = format12->t##loss; \
-            format20->t##shift = format12->t##shift;
-        UPDATEFMT20(R);
-        UPDATEFMT20(G);
-        UPDATEFMT20(B);
-        UPDATEFMT20(A);
-        #undef UPDATEFMT20
     }
 }