SDL: Fixed #5143 - software Render Jitter in rotation

From ceb09ee74099a0f64695b57246e4a1e8724a536b Mon Sep 17 00:00:00 2001
From: Sylvain <[EMAIL REDACTED]>
Date: Thu, 17 Feb 2022 23:11:02 +0100
Subject: [PATCH] Fixed #5143 - software Render Jitter in rotation better
 precision calculating rotated coordinates and interpolation

---
 src/render/software/SDL_render_sw.c |  55 +++------
 src/render/software/SDL_rotate.c    | 178 +++++++++++++++++-----------
 src/render/software/SDL_rotate.h    |   6 +-
 3 files changed, 124 insertions(+), 115 deletions(-)

diff --git a/src/render/software/SDL_render_sw.c b/src/render/software/SDL_render_sw.c
index 4fabcee48a8..a1af12cd677 100644
--- a/src/render/software/SDL_render_sw.c
+++ b/src/render/software/SDL_render_sw.c
@@ -435,59 +435,32 @@ SW_RenderCopyEx(SDL_Renderer * renderer, SDL_Surface *surface, SDL_Texture * tex
     SDL_SetSurfaceBlendMode(src_clone, blendmode);
 
     if (!retval) {
-        int dstwidth, dstheight;
+        SDL_Rect rect_dest;
         double cangle, sangle;
 
-        SDLgfx_rotozoomSurfaceSizeTrig(tmp_rect.w, tmp_rect.h, angle, &dstwidth, &dstheight, &cangle, &sangle);
-        src_rotated = SDLgfx_rotateSurface(src_clone, angle, dstwidth/2, dstheight/2, (texture->scaleMode == SDL_ScaleModeNearest) ? 0 : 1, flip & SDL_FLIP_HORIZONTAL, flip & SDL_FLIP_VERTICAL, dstwidth, dstheight, cangle, sangle);
+        SDLgfx_rotozoomSurfaceSizeTrig(tmp_rect.w, tmp_rect.h, angle, center,
+                &rect_dest, &cangle, &sangle);
+        src_rotated = SDLgfx_rotateSurface(src_clone, angle,
+                (texture->scaleMode == SDL_ScaleModeNearest) ? 0 : 1, flip & SDL_FLIP_HORIZONTAL, flip & SDL_FLIP_VERTICAL,
+                &rect_dest, cangle, sangle, center);
         if (src_rotated == NULL) {
             retval = -1;
         }
         if (!retval && mask != NULL) {
             /* The mask needed for the NONE blend mode gets rotated with the same parameters. */
-            mask_rotated = SDLgfx_rotateSurface(mask, angle, dstwidth/2, dstheight/2, SDL_FALSE, 0, 0, dstwidth, dstheight, cangle, sangle);
+            mask_rotated = SDLgfx_rotateSurface(mask, angle,
+                    SDL_FALSE, 0, 0,
+                    &rect_dest, cangle, sangle, center);
             if (mask_rotated == NULL) {
                 retval = -1;
             }
         }
         if (!retval) {
-            double abscenterx, abscentery;
-            double px, py, p1x, p1y, p2x, p2y, p3x, p3y, p4x, p4y;
-
-            /* Find out where the new origin is by rotating the four final_rect points around the center and then taking the extremes */
-            abscenterx = final_rect->x + center->x;
-            abscentery = final_rect->y + center->y;
-            /* Compensate the angle inversion to match the behaviour of the other backends */
-            sangle = -sangle;
-
-            /* Top Left */
-            px = final_rect->x - abscenterx;
-            py = final_rect->y - abscentery;
-            p1x = px * cangle - py * sangle + abscenterx;
-            p1y = px * sangle + py * cangle + abscentery;
-
-            /* Top Right */
-            px = final_rect->x + final_rect->w - abscenterx;
-            py = final_rect->y - abscentery;
-            p2x = px * cangle - py * sangle + abscenterx;
-            p2y = px * sangle + py * cangle + abscentery;
-
-            /* Bottom Left */
-            px = final_rect->x - abscenterx;
-            py = final_rect->y + final_rect->h - abscentery;
-            p3x = px * cangle - py * sangle + abscenterx;
-            p3y = px * sangle + py * cangle + abscentery;
-
-            /* Bottom Right */
-            px = final_rect->x + final_rect->w - abscenterx;
-            py = final_rect->y + final_rect->h - abscentery;
-            p4x = px * cangle - py * sangle + abscenterx;
-            p4y = px * sangle + py * cangle + abscentery;
-
-            tmp_rect.x = (int)MIN(MIN(p1x, p2x), MIN(p3x, p4x));
-            tmp_rect.y = (int)MIN(MIN(p1y, p2y), MIN(p3y, p4y));
-            tmp_rect.w = dstwidth;
-            tmp_rect.h = dstheight;
+
+            tmp_rect.x = final_rect->x + rect_dest.x;
+            tmp_rect.y = final_rect->y + rect_dest.y;
+            tmp_rect.w = rect_dest.w;
+            tmp_rect.h = rect_dest.h;
 
             /* The NONE blend mode needs some special care with non-opaque surfaces.
              * Other blend modes or opaque surfaces can be blitted directly.
diff --git a/src/render/software/SDL_rotate.c b/src/render/software/SDL_rotate.c
index cd73bcdc412..2b9dec81e5e 100644
--- a/src/render/software/SDL_rotate.c
+++ b/src/render/software/SDL_rotate.c
@@ -82,7 +82,7 @@ to a situation where the program can segfault.
 \brief Returns colorkey info for a surface
 */
 static Uint32
-_colorkey(SDL_Surface *src)
+get_colorkey(SDL_Surface *src)
 {
     Uint32 key = 0;
     if (SDL_HasColorKey(src)) {
@@ -91,6 +91,18 @@ _colorkey(SDL_Surface *src)
     return key;
 }
 
+/* rotate (sx, sy) by (angle, center) into (dx, dy) */
+static void
+rotate(double sx, double sy, double sinangle, double cosangle, const SDL_FPoint *center, double *dx, double *dy) {
+    sx -= center->x;
+    sy -= center->y;
+
+    *dx = cosangle * sx - sinangle * sy;
+    *dy = sinangle * sx + cosangle * sy;
+
+    *dx += center->x;
+    *dy += center->y;
+}
 
 /* !
 \brief Internal target surface sizing function for rotations with trig result return.
@@ -105,49 +117,61 @@ _colorkey(SDL_Surface *src)
 
 */
 void
-SDLgfx_rotozoomSurfaceSizeTrig(int width, int height, double angle,
-                               int *dstwidth, int *dstheight,
-                               double *cangle, double *sangle)
+SDLgfx_rotozoomSurfaceSizeTrig(int width, int height, double angle, const SDL_FPoint *center,
+                               SDL_Rect *rect_dest, double *cangle, double *sangle)
 {
-    /* The trig code below gets the wrong size (due to FP inaccuracy?) when angle is a multiple of 90 degrees */
-    int angle90 = (int)(angle/90);
-    if(angle90 == angle/90) { /* if the angle is a multiple of 90 degrees */
-        angle90 %= 4;
-        if(angle90 < 0) angle90 += 4; /* 0:0 deg, 1:90 deg, 2:180 deg, 3:270 deg */
-        if(angle90 & 1) {
-            *dstwidth  = height;
-            *dstheight = width;
-            *cangle = 0;
-            *sangle = angle90 == 1 ? -1 : 1; /* reversed because our rotations are clockwise */
-        } else {
-            *dstwidth  = width;
-            *dstheight = height;
-            *cangle = angle90 == 0 ? 1 : -1;
-            *sangle = 0;
+    int minx, maxx, miny, maxy;
+    double radangle;
+    double x0, x1, x2, x3;
+    double y0, y1, y2, y3;
+    double sinangle;
+    double cosangle;
+
+    radangle = angle * (M_PI / 180.0);
+    sinangle = SDL_sin(radangle);
+    cosangle = SDL_cos(radangle);
+
+    /*
+     * Determine destination width and height by rotating a source box, at pixel center
+     */
+    rotate(0.5,         0.5,          sinangle, cosangle, center, &x0, &y0);
+    rotate(width - 0.5, 0.5,          sinangle, cosangle, center, &x1, &y1);
+    rotate(0.5,         height - 0.5, sinangle, cosangle, center, &x2, &y2);
+    rotate(width - 0.5, height - 0.5, sinangle, cosangle, center, &x3, &y3);
+
+    minx = SDL_floor( SDL_min( SDL_min(x0, x1), SDL_min(x2, x3) ) );
+    maxx = SDL_ceil(  SDL_max( SDL_max(x0, x1), SDL_max(x2, x3) ) );
+
+    miny = SDL_floor( SDL_min( SDL_min(y0, y1), SDL_min(y2, y3) ) );
+    maxy = SDL_ceil(  SDL_max( SDL_max(y0, y1), SDL_max(y2, y3) ) );
+
+    rect_dest->w = maxx - minx;
+    rect_dest->h = maxy - miny;
+    rect_dest->x = minx;
+    rect_dest->y = miny;
+
+    /* reverse the angle because our rotations are clockwise */
+    *sangle = -sinangle;
+    *cangle = cosangle;
+
+    {
+        /* The trig code below gets the wrong size (due to FP inaccuracy?) when angle is a multiple of 90 degrees */
+        int angle90 = (int)(angle/90);
+        if(angle90 == angle/90) { /* if the angle is a multiple of 90 degrees */
+            angle90 %= 4;
+            if(angle90 < 0) angle90 += 4; /* 0:0 deg, 1:90 deg, 2:180 deg, 3:270 deg */
+            if(angle90 & 1) {
+                rect_dest->w  = height;
+                rect_dest->h = width;
+                *cangle = 0;
+                *sangle = angle90 == 1 ? -1 : 1; /* reversed because our rotations are clockwise */
+            } else {
+                rect_dest->w  = width;
+                rect_dest->h = height;
+                *cangle = angle90 == 0 ? 1 : -1;
+                *sangle = 0;
+            }
         }
-    } else {
-        double x, y, cx, cy, sx, sy;
-        double radangle;
-        double dstwidth_max, dstheight_max;
-        /*
-        * Determine destination width and height by rotating a centered source box
-        */
-        radangle = angle * (M_PI / -180.0); /* reverse the angle because our rotations are clockwise */
-        *sangle = SDL_sin(radangle);
-        *cangle = SDL_cos(radangle);
-        x = (double)width;
-        y = (double)height;
-        cx = *cangle * x;
-        cy = *cangle * y;
-        sx = *sangle * x;
-        sy = *sangle * y;
-
-        dstwidth_max = MAX(
-            MAX(MAX(MAX(SDL_fabs(cx + sy), SDL_fabs(cx - sy)), SDL_fabs(-cx + sy)), SDL_fabs(-cx - sy)), 1);
-        dstheight_max = MAX(
-            MAX(MAX(MAX(SDL_fabs(sx + cy), SDL_fabs(sx - cy)), SDL_fabs(-sx + cy)), SDL_fabs(-sx - cy)), 1);
-        *dstwidth = SDL_round(dstwidth_max);
-        *dstheight = SDL_round(dstheight_max);
     }
 }
 
@@ -220,48 +244,55 @@ Assumes dst surface was allocated with the correct dimensions.
 
 \param src Source surface.
 \param dst Destination surface.
-\param cx Horizontal center coordinate.
-\param cy Vertical center coordinate.
 \param isin Integer version of sine of angle.
 \param icos Integer version of cosine of angle.
 \param flipx Flag indicating horizontal mirroring should be applied.
 \param flipy Flag indicating vertical mirroring should be applied.
 \param smooth Flag indicating anti-aliasing should be used.
+\param center true center.
 */
 static void
-_transformSurfaceRGBA(SDL_Surface * src, SDL_Surface * dst, int cx, int cy, int isin, int icos, int flipx, int flipy, int smooth)
+transformSurfaceRGBA(SDL_Surface * src, SDL_Surface * dst, int isin, int icos,
+        int flipx, int flipy, int smooth,
+        const SDL_Rect *rect_dest,
+        const SDL_FPoint *center)
 {
-    int x, y, t1, t2, dx, dy, xd, yd, sdx, sdy, ax, ay, ex, ey, sw, sh;
+    int sw, sh;
+    int cx, cy;
     tColorRGBA c00, c01, c10, c11, cswap;
     tColorRGBA *pc, *sp;
     int gap;
+    const int fp_half = (1<<15);
 
     /*
     * Variable setup
     */
-    xd = ((src->w - dst->w) << 15);
-    yd = ((src->h - dst->h) << 15);
-    ax = (cx << 16) - (icos * cx);
-    ay = (cy << 16) - (isin * cx);
     sw = src->w - 1;
     sh = src->h - 1;
     pc = (tColorRGBA*) dst->pixels;
     gap = dst->pitch - dst->w * 4;
+    cx = (int)(center->x * 65536.0);
+    cy = (int)(center->y * 65536.0);
 
     /*
     * Switch between interpolating and non-interpolating code
     */
     if (smooth) {
+        int y;
         for (y = 0; y < dst->h; y++) {
-            dy = cy - y;
-            sdx = (ax + (isin * dy)) + xd;
-            sdy = (ay - (icos * dy)) + yd;
+            int x;
+            double src_x = (rect_dest->x + 0 + 0.5 - center->x);
+            double src_y = (rect_dest->y + y + 0.5 - center->y);
+            int sdx = (icos * src_x - isin * src_y) + cx - fp_half;
+            int sdy = (isin * src_x + icos * src_y) + cy - fp_half;
             for (x = 0; x < dst->w; x++) {
-                dx = (sdx >> 16);
-                dy = (sdy >> 16);
+                int dx = (sdx >> 16);
+                int dy = (sdy >> 16);
                 if (flipx) dx = sw - dx;
                 if (flipy) dy = sh - dy;
                 if ((dx > -1) && (dy > -1) && (dx < (src->w-1)) && (dy < (src->h-1))) {
+                    int ex, ey;
+                    int t1, t2;
                     sp = (tColorRGBA *) ((Uint8 *) src->pixels + src->pitch * dy) + dx;
                     c00 = *sp;
                     sp += 1;
@@ -303,13 +334,16 @@ _transformSurfaceRGBA(SDL_Surface * src, SDL_Surface * dst, int cx, int cy, int
             pc = (tColorRGBA *) ((Uint8 *) pc + gap);
         }
     } else {
+        int y;
         for (y = 0; y < dst->h; y++) {
-            dy = cy - y;
-            sdx = (ax + (isin * dy)) + xd;
-            sdy = (ay - (icos * dy)) + yd;
+            int x;
+            double src_x = (rect_dest->x + 0 + 0.5 - center->x);
+            double src_y = (rect_dest->y + y + 0.5 - center->y);
+            int sdx = (icos * src_x - isin * src_y) + cx - fp_half;
+            int sdy = (isin * src_x + icos * src_y) + cy - fp_half;
             for (x = 0; x < dst->w; x++) {
-                dx = (sdx >> 16);
-                dy = (sdy >> 16);
+                int dx = (sdx >> 16);
+                int dy = (sdy >> 16);
                 if ((unsigned)dx < (unsigned)src->w && (unsigned)dy < (unsigned)src->h) {
                     if(flipx) dx = sw - dx;
                     if(flipy) dy = sh - dy;
@@ -361,7 +395,7 @@ transformSurfaceY(SDL_Surface * src, SDL_Surface * dst, int cx, int cy, int isin
     /*
     * Clear surface to colorkey
     */
-    SDL_memset(pc, (int)(_colorkey(src) & 0xff), dst->pitch * dst->h);
+    SDL_memset(pc, (int)(get_colorkey(src) & 0xff), dst->pitch * dst->h);
     /*
     * Iterate through destination surface
     */
@@ -400,21 +434,21 @@ When using the NONE and MOD modes, color and alpha modulation must be applied be
 
 \param src The surface to rotozoom.
 \param angle The angle to rotate in degrees.
-\param centerx The horizontal coordinate of the center of rotation
 \param zoomy The vertical coordinate of the center of rotation
 \param smooth Antialiasing flag; set to SMOOTHING_ON to enable.
 \param flipx Set to 1 to flip the image horizontally
 \param flipy Set to 1 to flip the image vertically
-\param dstwidth The destination surface width
-\param dstheight The destination surface height
+\param rect_dest The destination rect bounding box
 \param cangle The angle cosine
 \param sangle The angle sine
+\param center The true coordinate of the center of rotation
 \return The new rotated surface.
 
 */
 
 SDL_Surface *
-SDLgfx_rotateSurface(SDL_Surface * src, double angle, int centerx, int centery, int smooth, int flipx, int flipy, int dstwidth, int dstheight, double cangle, double sangle)
+SDLgfx_rotateSurface(SDL_Surface * src, double angle, int smooth, int flipx, int flipy,
+        const SDL_Rect *rect_dest, double cangle, double sangle, const SDL_FPoint *center)
 {
     SDL_Surface *rz_dst;
     int is8bit, angle90;
@@ -447,7 +481,7 @@ SDLgfx_rotateSurface(SDL_Surface * src, double angle, int centerx, int centery,
     rz_dst = NULL;
     if (is8bit) {
         /* Target surface is 8 bit */
-        rz_dst = SDL_CreateRGBSurface(0, dstwidth, dstheight + GUARD_ROWS, 8, 0, 0, 0, 0);
+        rz_dst = SDL_CreateRGBSurface(0, rect_dest->w, rect_dest->h + GUARD_ROWS, 8, 0, 0, 0, 0);
         if (rz_dst != NULL) {
             for (i = 0; i < src->format->palette->ncolors; i++) {
                 rz_dst->format->palette->colors[i] = src->format->palette->colors[i];
@@ -456,7 +490,7 @@ SDLgfx_rotateSurface(SDL_Surface * src, double angle, int centerx, int centery,
         }
     } else {
         /* Target surface is 32 bit with source RGBA ordering */
-        rz_dst = SDL_CreateRGBSurface(0, dstwidth, dstheight + GUARD_ROWS, 32,
+        rz_dst = SDL_CreateRGBSurface(0, rect_dest->w, rect_dest->h + GUARD_ROWS, 32,
                                       src->format->Rmask, src->format->Gmask,
                                       src->format->Bmask, src->format->Amask);
     }
@@ -466,7 +500,7 @@ SDLgfx_rotateSurface(SDL_Surface * src, double angle, int centerx, int centery,
         return NULL;
 
     /* Adjust for guard rows */
-    rz_dst->h = dstheight;
+    rz_dst->h = rect_dest->h;
 
     SDL_GetSurfaceBlendMode(src, &blendmode);
 
@@ -497,7 +531,7 @@ SDLgfx_rotateSurface(SDL_Surface * src, double angle, int centerx, int centery,
     }
 
     /* check if the rotation is a multiple of 90 degrees so we can take a fast path and also somewhat reduce
-     * the off-by-one problem in _transformSurfaceRGBA that expresses itself when the rotation is near
+     * the off-by-one problem in transformSurfaceRGBA that expresses itself when the rotation is near
      * multiples of 90 degrees.
      */
     angle90 = (int)(angle/90);
@@ -513,7 +547,7 @@ SDLgfx_rotateSurface(SDL_Surface * src, double angle, int centerx, int centery,
         if(angle90 >= 0) {
             transformSurfaceY90(src, rz_dst, angle90, flipx, flipy);
         } else {
-            transformSurfaceY(src, rz_dst, centerx, centery, (int)sangleinv, (int)cangleinv,
+            transformSurfaceY(src, rz_dst, rect_dest->w/2, rect_dest->h/2, (int)sangleinv, (int)cangleinv,
                               flipx, flipy);
         }
     } else {
@@ -521,8 +555,8 @@ SDLgfx_rotateSurface(SDL_Surface * src, double angle, int centerx, int centery,
         if (angle90 >= 0) {
             transformSurfaceRGBA90(src, rz_dst, angle90, flipx, flipy);
         } else {
-            _transformSurfaceRGBA(src, rz_dst, centerx, centery, (int)sangleinv, (int)cangleinv,
-                                  flipx, flipy, smooth);
+            transformSurfaceRGBA(src, rz_dst, (int)sangleinv, (int)cangleinv,
+                                  flipx, flipy, smooth, rect_dest, center);
         }
     }
 
diff --git a/src/render/software/SDL_rotate.h b/src/render/software/SDL_rotate.h
index e1746c92c85..171baa5dd5e 100644
--- a/src/render/software/SDL_rotate.h
+++ b/src/render/software/SDL_rotate.h
@@ -26,7 +26,9 @@
 #define MIN(a,b)    (((a) < (b)) ? (a) : (b))
 #endif
 
-extern SDL_Surface *SDLgfx_rotateSurface(SDL_Surface * src, double angle, int centerx, int centery, int smooth, int flipx, int flipy, int dstwidth, int dstheight, double cangle, double sangle);
-extern void SDLgfx_rotozoomSurfaceSizeTrig(int width, int height, double angle, int *dstwidth, int *dstheight, double *cangle, double *sangle);
+extern SDL_Surface *SDLgfx_rotateSurface(SDL_Surface * src, double angle, int smooth, int flipx, int flipy,
+        const SDL_Rect *rect_dest, double cangle, double sangle, const SDL_FPoint *center);
+extern void SDLgfx_rotozoomSurfaceSizeTrig(int width, int height, double angle, const SDL_FPoint *center,
+        SDL_Rect *rect_dest, double *cangle, double *sangle);
 
 #endif /* SDL_rotate_h_ */