sdl12-compat: SaveDestAlpha: clamp dest rectangle correctly.

From 61e247dfd57b8aebdc3078f9c1657079b7e66721 Mon Sep 17 00:00:00 2001
From: "Ryan C. Gordon" <[EMAIL REDACTED]>
Date: Mon, 24 Oct 2022 10:53:44 -0400
Subject: [PATCH] SaveDestAlpha: clamp dest rectangle correctly.

SDL blits only care about the destination x and y, ignoring the w and h,
and then we need to clamp on top of that anyhow, in case the x and y are
out of bounds.

Reference Issue #244.
---
 src/SDL12_compat.c | 73 +++++++++++++++++++++++++---------------------
 1 file changed, 40 insertions(+), 33 deletions(-)

diff --git a/src/SDL12_compat.c b/src/SDL12_compat.c
index eae326ca6..70427258a 100644
--- a/src/SDL12_compat.c
+++ b/src/SDL12_compat.c
@@ -6232,7 +6232,7 @@ SDL_GetVideoSurface(void)
 }
 
 static int
-SaveDestAlpha(SDL12_Surface *src12, SDL12_Surface *dst12, const SDL12_Rect *dstrect12, Uint8 **retval)
+SaveDestAlpha(SDL12_Surface *src12, SDL12_Surface *dst12, SDL_Rect *dstrect20, Uint8 **retval)
 {
     /* The 1.2 docs say this:
      * RGBA->RGBA:
@@ -6245,24 +6245,15 @@ SaveDestAlpha(SDL12_Surface *src12, SDL12_Surface *dst12, const SDL12_Rect *dstr
     Uint8 *dstalpha = NULL;
     const SDL_bool save_dstalpha = ((src12->flags & SDL12_SRCALPHA) && dst12->format->Amask && ((src12->format->alpha != 255) || src12->format->Amask)) ? SDL_TRUE : SDL_FALSE;
 
-    if (save_dstalpha) {
+    if (save_dstalpha && (dstrect20->w > 0) && (dstrect20->h > 0)) {
         const Uint32 amask = dst12->format->Amask;
         const Uint32 ashift = dst12->format->Ashift;
         const Uint16 pitch = dst12->pitch;
-        SDL12_Rect fullrect;
         Uint8 *dptr;
         int x, y, w, h;
 
-        if (!dstrect12) {
-            fullrect.x = 0;
-            fullrect.y = 0;
-            fullrect.w = dst12->w;
-            fullrect.h = dst12->h;
-            dstrect12 = &fullrect;
-        }
-
-        w = dstrect12->w;
-        h = dstrect12->h;
+        w = dstrect20->w;
+        h = dstrect20->h;
 
         dstalpha = (Uint8 *) SDL20_malloc(w * h);
         if (!dstalpha) {
@@ -6273,7 +6264,7 @@ SaveDestAlpha(SDL12_Surface *src12, SDL12_Surface *dst12, const SDL12_Rect *dstr
 
         if (dst12->format->BytesPerPixel == 2) {
             const Uint16 *sptr = (const Uint16 *) dst12->pixels;
-            sptr += ((dst12->pitch / 2) * dstrect12->y) + dstrect12->x;
+            sptr += ((dst12->pitch / 2) * dstrect20->y) + dstrect20->x;
             for (y = 0; y < h; y++) {
                 for (x = 0; x < w; x++) {
                     *(dptr++) = (Uint8) ((sptr[x] & amask) >> ashift);
@@ -6282,7 +6273,7 @@ SaveDestAlpha(SDL12_Surface *src12, SDL12_Surface *dst12, const SDL12_Rect *dstr
             }
         } else if (dst12->format->BytesPerPixel == 4) {
             const Uint32 *sptr = (const Uint32 *) dst12->pixels;
-            sptr += ((dst12->pitch / 4) * dstrect12->y) + dstrect12->x;
+            sptr += ((dst12->pitch / 4) * dstrect20->y) + dstrect20->x;
             for (y = 0; y < h; y++) {
                 for (x = 0; x < w; x++) {
                     *(dptr++) = (Uint8) ((sptr[x] & amask) >> ashift);
@@ -6299,30 +6290,21 @@ SaveDestAlpha(SDL12_Surface *src12, SDL12_Surface *dst12, const SDL12_Rect *dstr
 }
 
 static void
-RestoreDestAlpha(SDL12_Surface *dst12, Uint8 *dstalpha, const SDL12_Rect *dstrect12)
+RestoreDestAlpha(SDL12_Surface *dst12, Uint8 *dstalpha, const SDL_Rect *dstrect20)
 {
     if (dstalpha) {
         const Uint8 *sptr = dstalpha;
         const Uint32 amask = dst12->format->Amask;
         const Uint32 ashift = dst12->format->Ashift;
         const Uint16 pitch = dst12->pitch;
-        SDL12_Rect fullrect;
         int x, y, w, h;
 
-        if (!dstrect12) {
-            fullrect.x = 0;
-            fullrect.y = 0;
-            fullrect.w = dst12->w;
-            fullrect.h = dst12->h;
-            dstrect12 = &fullrect;
-        }
-
-        w = dstrect12->w;
-        h = dstrect12->h;
+        w = dstrect20->w;
+        h = dstrect20->h;
 
         if (dst12->format->BytesPerPixel == 2) {
             Uint16 *dptr = (Uint16 *) dst12->pixels;
-            dptr += ((dst12->pitch / 2) * dstrect12->y) + dstrect12->x;
+            dptr += ((dst12->pitch / 2) * dstrect20->y) + dstrect20->x;
             for (y = 0; y < h; y++) {
                 for (x = 0; x < w; x++) {
                     dptr[x] = (Uint16) ((dptr[x] & ~amask) | ((((Uint16) *(sptr++)) << ashift) & amask));
@@ -6331,7 +6313,7 @@ RestoreDestAlpha(SDL12_Surface *dst12, Uint8 *dstalpha, const SDL12_Rect *dstrec
             }
         } else if (dst12->format->BytesPerPixel == 4) {
             Uint32 *dptr = (Uint32 *) dst12->pixels;
-            dptr += ((dst12->pitch / 4) * dstrect12->y) + dstrect12->x;
+            dptr += ((dst12->pitch / 4) * dstrect20->y) + dstrect20->x;
             for (y = 0; y < h; y++) {
                 for (x = 0; x < w; x++) {
                     dptr[x] = (dptr[x] & ~amask) | ((((Uint32) *(sptr++)) << ashift) & amask);
@@ -6345,6 +6327,27 @@ RestoreDestAlpha(SDL12_Surface *dst12, Uint8 *dstalpha, const SDL12_Rect *dstrec
     }
 }
 
+static void
+PrepBlitDestRect(SDL_Rect *dstrect20, SDL12_Surface *dst12, const SDL12_Rect *dstrect12)
+{
+    /* dstrect12 w and h is ignored, SDL 1.2 only cares about position. */
+    dstrect20->w = dst12->w;
+    dstrect20->h = dst12->h;
+
+    if (dstrect12) {
+        SDL_Rect fulldstrect20;
+        fulldstrect20.x = fulldstrect20.y = 0;
+        fulldstrect20.w = dst12->w;
+        fulldstrect20.h = dst12->h;
+        dstrect20->x = dstrect12->x;
+        dstrect20->y = dstrect12->y;
+        SDL20_IntersectRect(&fulldstrect20, dstrect20, dstrect20);
+    } else {
+        dstrect20->x = 0;
+        dstrect20->y = 0;
+    }
+}
+
 DECLSPEC12 int SDLCALL
 SDL_UpperBlit(SDL12_Surface *src12, SDL12_Rect *srcrect12, SDL12_Surface *dst12, SDL12_Rect *dstrect12)
 {
@@ -6352,11 +6355,13 @@ SDL_UpperBlit(SDL12_Surface *src12, SDL12_Rect *srcrect12, SDL12_Surface *dst12,
     SDL_Rect srcrect20, dstrect20;
     int retval;
 
+    PrepBlitDestRect(&dstrect20, dst12, dstrect12);
+
     if ((src12 == NULL) || (dst12 == NULL)) {
         return SDL20_SetError("SDL_UpperBlit: passed a NULL surface");
     } else if ((src12->pixels == NULL) || (dst12->pixels == NULL)) {
         return SDL20_SetError("SDL_UpperBlit: passed a surface with NULL pixels");
-    } else if (SaveDestAlpha(src12, dst12, dstrect12, &dstalpha) < 0) {
+    } else if (SaveDestAlpha(src12, dst12, &dstrect20, &dstalpha) < 0) {
         return -1;
     }
 
@@ -6365,7 +6370,7 @@ SDL_UpperBlit(SDL12_Surface *src12, SDL12_Rect *srcrect12, SDL12_Surface *dst12,
                              dst12->surface20,
                              dstrect12 ? Rect12to20(dstrect12, &dstrect20) : NULL);
 
-    RestoreDestAlpha(dst12, dstalpha, dstrect12);
+    RestoreDestAlpha(dst12, dstalpha, &dstrect20);
 
     if (dstrect12) {
         Rect20to12(&dstrect20, dstrect12);
@@ -6381,7 +6386,9 @@ SDL_LowerBlit(SDL12_Surface *src12, SDL12_Rect *srcrect12, SDL12_Surface *dst12,
     SDL_Rect srcrect20, dstrect20;
     int retval;
 
-    if (SaveDestAlpha(src12, dst12, dstrect12, &dstalpha) < 0) {
+    PrepBlitDestRect(&dstrect20, dst12, dstrect12);
+
+    if (SaveDestAlpha(src12, dst12, &dstrect20, &dstalpha) < 0) {
         return -1;
     }
 
@@ -6390,7 +6397,7 @@ SDL_LowerBlit(SDL12_Surface *src12, SDL12_Rect *srcrect12, SDL12_Surface *dst12,
                              dst12->surface20,
                              dstrect12 ? Rect12to20(dstrect12, &dstrect20) : NULL);
 
-    RestoreDestAlpha(dst12, dstalpha, dstrect12);
+    RestoreDestAlpha(dst12, dstalpha, &dstrect20);
 
     if (srcrect12) {
         Rect20to12(&srcrect20, srcrect12);