SDL: test: Add test coverage for surface size overflows

From e974985998233504aad14d0e9d9af366db9b8028 Mon Sep 17 00:00:00 2001
From: Simon McVittie <[EMAIL REDACTED]>
Date: Mon, 13 Jun 2022 16:54:42 +0100
Subject: [PATCH] test: Add test coverage for surface size overflows

Signed-off-by: Simon McVittie <smcv@collabora.com>
---
 test/testautomation_surface.c | 176 +++++++++++++++++++++++++++++++++-
 1 file changed, 175 insertions(+), 1 deletion(-)

diff --git a/test/testautomation_surface.c b/test/testautomation_surface.c
index d159c1f40e9..63d4c294dd6 100644
--- a/test/testautomation_surface.c
+++ b/test/testautomation_surface.c
@@ -593,6 +593,177 @@ surface_testBlitBlendLoop(void *arg) {
 
 }
 
+int
+surface_testOverflow(void *arg)
+{
+    char buf[1024];
+    const char *expectedError;
+    SDL_Surface *surface;
+
+    SDL_memset(buf, '\0', sizeof(buf));
+
+    expectedError = "Parameter 'width' is invalid";
+    surface = SDL_CreateRGBSurfaceWithFormat(0, -3, 100, 8, SDL_PIXELFORMAT_INDEX8);
+    SDLTest_AssertCheck(surface == NULL, "Should detect negative width");
+    SDLTest_AssertCheck(SDL_strcmp(SDL_GetError(), expectedError) == 0,
+                        "Expected \"%s\", got \"%s\"", expectedError, SDL_GetError());
+    surface = SDL_CreateRGBSurfaceWithFormatFrom(buf, -1, 1, 8, 4, SDL_PIXELFORMAT_INDEX8);
+    SDLTest_AssertCheck(surface == NULL, "Should detect negative width");
+    SDLTest_AssertCheck(SDL_strcmp(SDL_GetError(), expectedError) == 0,
+                        "Expected \"%s\", got \"%s\"", expectedError, SDL_GetError());
+    surface = SDL_CreateRGBSurfaceFrom(buf, -1, 1, 32, 4, 0xFF000000, 0x00FF0000, 0x0000FF00, 0x000000FF);
+    SDLTest_AssertCheck(surface == NULL, "Should detect negative width");
+    SDLTest_AssertCheck(SDL_strcmp(SDL_GetError(), expectedError) == 0,
+                        "Expected \"%s\", got \"%s\"", expectedError, SDL_GetError());
+
+    expectedError = "Parameter 'height' is invalid";
+    surface = SDL_CreateRGBSurfaceWithFormat(0, 100, -3, 8, SDL_PIXELFORMAT_INDEX8);
+    SDLTest_AssertCheck(surface == NULL, "Should detect negative height");
+    SDLTest_AssertCheck(SDL_strcmp(SDL_GetError(), expectedError) == 0,
+                        "Expected \"%s\", got \"%s\"", expectedError, SDL_GetError());
+    surface = SDL_CreateRGBSurfaceWithFormatFrom(buf, 1, -1, 8, 4, SDL_PIXELFORMAT_INDEX8);
+    SDLTest_AssertCheck(surface == NULL, "Should detect negative height");
+    SDLTest_AssertCheck(SDL_strcmp(SDL_GetError(), expectedError) == 0,
+                        "Expected \"%s\", got \"%s\"", expectedError, SDL_GetError());
+    surface = SDL_CreateRGBSurfaceFrom(buf, 1, -1, 32, 4, 0xFF000000, 0x00FF0000, 0x0000FF00, 0x000000FF);
+    SDLTest_AssertCheck(surface == NULL, "Should detect negative height");
+    SDLTest_AssertCheck(SDL_strcmp(SDL_GetError(), expectedError) == 0,
+                        "Expected \"%s\", got \"%s\"", expectedError, SDL_GetError());
+
+    expectedError = "Parameter 'pitch' is invalid";
+    surface = SDL_CreateRGBSurfaceWithFormatFrom(buf, 4, 1, 8, -1, SDL_PIXELFORMAT_INDEX8);
+    SDLTest_AssertCheck(surface == NULL, "Should detect negative pitch");
+    SDLTest_AssertCheck(SDL_strcmp(SDL_GetError(), expectedError) == 0,
+                        "Expected \"%s\", got \"%s\"", expectedError, SDL_GetError());
+    surface = SDL_CreateRGBSurfaceFrom(buf, 1, 1, 32, -1, 0xFF000000, 0x00FF0000, 0x0000FF00, 0x000000FF);
+    SDLTest_AssertCheck(surface == NULL, "Should detect negative pitch");
+    SDLTest_AssertCheck(SDL_strcmp(SDL_GetError(), expectedError) == 0,
+                        "Expected \"%s\", got \"%s\"", expectedError, SDL_GetError());
+
+    /* Less than 1 byte per pixel: the pitch can legitimately be less than
+     * the width, but it must be enough to hold the appropriate number of
+     * bits per pixel. SDL_PIXELFORMAT_INDEX4* needs 1 byte per 2 pixels. */
+    surface = SDL_CreateRGBSurfaceWithFormatFrom(buf, 6, 1, 4, 3, SDL_PIXELFORMAT_INDEX4LSB);
+    SDLTest_AssertCheck(surface != NULL, "6px * 4 bits per px fits in 3 bytes: %s",
+                        surface != NULL ? "(success)" : SDL_GetError());
+    SDL_FreeSurface(surface);
+    surface = SDL_CreateRGBSurfaceFrom(buf, 6, 1, 4, 3, 0, 0, 0, 0);
+    SDLTest_AssertCheck(surface != NULL, "6px * 4 bits per px fits in 3 bytes: %s",
+                        surface != NULL ? "(success)" : SDL_GetError());
+    SDL_FreeSurface(surface);
+
+    surface = SDL_CreateRGBSurfaceWithFormatFrom(buf, 7, 1, 4, 3, SDL_PIXELFORMAT_INDEX4LSB);
+    SDLTest_AssertCheck(surface == NULL, "Should detect pitch < width * bpp");
+    SDLTest_AssertCheck(SDL_strcmp(SDL_GetError(), expectedError) == 0,
+                        "Expected \"%s\", got \"%s\"", expectedError, SDL_GetError());
+    surface = SDL_CreateRGBSurfaceFrom(buf, 7, 1, 4, 3, 0, 0, 0, 0);
+    SDLTest_AssertCheck(surface == NULL, "Should detect pitch < width * bpp");
+    SDLTest_AssertCheck(SDL_strcmp(SDL_GetError(), expectedError) == 0,
+                        "Expected \"%s\", got \"%s\"", expectedError, SDL_GetError());
+
+    surface = SDL_CreateRGBSurfaceWithFormatFrom(buf, 7, 1, 4, 4, SDL_PIXELFORMAT_INDEX4LSB);
+    SDLTest_AssertCheck(surface != NULL, "7px * 4 bits per px fits in 4 bytes: %s",
+                        surface != NULL ? "(success)" : SDL_GetError());
+    SDL_FreeSurface(surface);
+    surface = SDL_CreateRGBSurfaceFrom(buf, 7, 1, 4, 4, 0, 0, 0, 0);
+    SDLTest_AssertCheck(surface != NULL, "7px * 4 bits per px fits in 4 bytes: %s",
+                        surface != NULL ? "(success)" : SDL_GetError());
+    SDL_FreeSurface(surface);
+
+    /* SDL_PIXELFORMAT_INDEX1* needs 1 byte per 8 pixels. */
+    surface = SDL_CreateRGBSurfaceWithFormatFrom(buf, 16, 1, 1, 2, SDL_PIXELFORMAT_INDEX1LSB);
+    SDLTest_AssertCheck(surface != NULL, "16px * 1 bit per px fits in 2 bytes: %s",
+                        surface != NULL ? "(success)" : SDL_GetError());
+    SDL_FreeSurface(surface);
+    surface = SDL_CreateRGBSurfaceFrom(buf, 16, 1, 1, 2, 0, 0, 0, 0);
+    SDLTest_AssertCheck(surface != NULL, "16px * 1 bit per px fits in 2 bytes: %s",
+                        surface != NULL ? "(success)" : SDL_GetError());
+    SDL_FreeSurface(surface);
+
+    surface = SDL_CreateRGBSurfaceWithFormatFrom(buf, 17, 1, 1, 2, SDL_PIXELFORMAT_INDEX1LSB);
+    SDLTest_AssertCheck(surface == NULL, "Should detect pitch < width * bpp");
+    SDLTest_AssertCheck(SDL_strcmp(SDL_GetError(), expectedError) == 0,
+                        "Expected \"%s\", got \"%s\"", expectedError, SDL_GetError());
+    surface = SDL_CreateRGBSurfaceFrom(buf, 17, 1, 1, 2, 0, 0, 0, 0);
+    SDLTest_AssertCheck(surface == NULL, "Should detect pitch < width * bpp");
+    SDLTest_AssertCheck(SDL_strcmp(SDL_GetError(), expectedError) == 0,
+                        "Expected \"%s\", got \"%s\"", expectedError, SDL_GetError());
+
+    surface = SDL_CreateRGBSurfaceWithFormatFrom(buf, 17, 1, 1, 3, SDL_PIXELFORMAT_INDEX1LSB);
+    SDLTest_AssertCheck(surface != NULL, "17px * 1 bit per px fits in 3 bytes: %s",
+                        surface != NULL ? "(success)" : SDL_GetError());
+    SDL_FreeSurface(surface);
+    surface = SDL_CreateRGBSurfaceFrom(buf, 7, 1, 1, 3, 0, 0, 0, 0);
+    SDLTest_AssertCheck(surface != NULL, "17px * 1 bit per px fits in 3 bytes: %s",
+                        surface != NULL ? "(success)" : SDL_GetError());
+    SDL_FreeSurface(surface);
+
+    /* SDL_PIXELFORMAT_INDEX8 and SDL_PIXELFORMAT_RGB332 require 1 byte per pixel. */
+    surface = SDL_CreateRGBSurfaceWithFormatFrom(buf, 5, 1, 8, 5, SDL_PIXELFORMAT_RGB332);
+    SDLTest_AssertCheck(surface != NULL, "5px * 8 bits per px fits in 5 bytes: %s",
+                        surface != NULL ? "(success)" : SDL_GetError());
+    SDL_FreeSurface(surface);
+    surface = SDL_CreateRGBSurfaceFrom(buf, 5, 1, 8, 5, 0, 0, 0, 0);
+    SDLTest_AssertCheck(surface != NULL, "5px * 8 bits per px fits in 5 bytes: %s",
+                        surface != NULL ? "(success)" : SDL_GetError());
+    SDL_FreeSurface(surface);
+
+    surface = SDL_CreateRGBSurfaceWithFormatFrom(buf, 6, 1, 8, 5, SDL_PIXELFORMAT_RGB332);
+    SDLTest_AssertCheck(surface == NULL, "Should detect pitch < width * bpp");
+    SDLTest_AssertCheck(SDL_strcmp(SDL_GetError(), expectedError) == 0,
+                        "Expected \"%s\", got \"%s\"", expectedError, SDL_GetError());
+    surface = SDL_CreateRGBSurfaceFrom(buf, 6, 1, 8, 5, 0, 0, 0, 0);
+    SDLTest_AssertCheck(surface == NULL, "Should detect pitch < width * bpp");
+    SDLTest_AssertCheck(SDL_strcmp(SDL_GetError(), expectedError) == 0,
+                        "Expected \"%s\", got \"%s\"", expectedError, SDL_GetError());
+
+    /* Everything else requires more than 1 byte per pixel, and rounds up
+     * each pixel to an integer number of bytes (e.g. RGB555 is really
+     * XRGB1555, with 1 bit per pixel wasted). */
+    surface = SDL_CreateRGBSurfaceWithFormatFrom(buf, 3, 1, 15, 6, SDL_PIXELFORMAT_RGB555);
+    SDLTest_AssertCheck(surface != NULL, "3px * 15 (really 16) bits per px fits in 6 bytes: %s",
+                        surface != NULL ? "(success)" : SDL_GetError());
+    SDL_FreeSurface(surface);
+    surface = SDL_CreateRGBSurfaceFrom(buf, 3, 1, 15, 6, 0, 0, 0, 0);
+    SDLTest_AssertCheck(surface != NULL, "5px * 15 (really 16) bits per px fits in 6 bytes: %s",
+                        surface != NULL ? "(success)" : SDL_GetError());
+    SDL_FreeSurface(surface);
+
+    surface = SDL_CreateRGBSurfaceWithFormatFrom(buf, 4, 1, 15, 6, SDL_PIXELFORMAT_RGB555);
+    SDLTest_AssertCheck(surface == NULL, "4px * 15 (really 16) bits per px doesn't fit in 6 bytes");
+    SDLTest_AssertCheck(SDL_strcmp(SDL_GetError(), expectedError) == 0,
+                        "Expected \"%s\", got \"%s\"", expectedError, SDL_GetError());
+    surface = SDL_CreateRGBSurfaceFrom(buf, 4, 1, 15, 6, 0, 0, 0, 0);
+    SDLTest_AssertCheck(surface == NULL, "4px * 15 (really 16) bits per px doesn't fit in 6 bytes");
+    SDLTest_AssertCheck(SDL_strcmp(SDL_GetError(), expectedError) == 0,
+                        "Expected \"%s\", got \"%s\"", expectedError, SDL_GetError());
+
+    if (sizeof (size_t) == 4 && sizeof (int) >= 4) {
+        expectedError = "Out of memory";
+        surface = SDL_CreateRGBSurfaceWithFormat(0, SDL_MAX_SINT32, 1, 8, SDL_PIXELFORMAT_INDEX8);
+        SDLTest_AssertCheck(surface == NULL, "Should detect overflow in width + alignment");
+        SDLTest_AssertCheck(SDL_strcmp(SDL_GetError(), expectedError) == 0,
+                            "Expected \"%s\", got \"%s\"", expectedError, SDL_GetError());
+        surface = SDL_CreateRGBSurfaceWithFormat(0, SDL_MAX_SINT32 / 2, 1, 32, SDL_PIXELFORMAT_ARGB8888);
+        SDLTest_AssertCheck(surface == NULL, "Should detect overflow in width * bytes per pixel");
+        SDLTest_AssertCheck(SDL_strcmp(SDL_GetError(), expectedError) == 0,
+                            "Expected \"%s\", got \"%s\"", expectedError, SDL_GetError());
+        surface = SDL_CreateRGBSurfaceWithFormat(0, (1 << 29) - 1, (1 << 29) - 1, 8, SDL_PIXELFORMAT_INDEX8);
+        SDLTest_AssertCheck(surface == NULL, "Should detect overflow in width * height");
+        SDLTest_AssertCheck(SDL_strcmp(SDL_GetError(), expectedError) == 0,
+                            "Expected \"%s\", got \"%s\"", expectedError, SDL_GetError());
+        surface = SDL_CreateRGBSurfaceWithFormat(0, (1 << 15) + 1, (1 << 15) + 1, 32, SDL_PIXELFORMAT_ARGB8888);
+        SDLTest_AssertCheck(surface == NULL, "Should detect overflow in width * height * bytes per pixel");
+        SDLTest_AssertCheck(SDL_strcmp(SDL_GetError(), expectedError) == 0,
+                            "Expected \"%s\", got \"%s\"", expectedError, SDL_GetError());
+    }
+    else {
+        SDLTest_Log("Can't easily overflow size_t on this platform");
+    }
+
+    return TEST_COMPLETED;
+}
+
 /* ================= Test References ================== */
 
 /* Surface test cases */
@@ -635,11 +806,14 @@ static const SDLTest_TestCaseReference surfaceTest11 =
 static const SDLTest_TestCaseReference surfaceTest12 =
         { (SDLTest_TestCaseFp)surface_testBlitBlendMod, "surface_testBlitBlendMod", "Tests blitting routines with mod blending mode.", TEST_ENABLED};
 
+static const SDLTest_TestCaseReference surfaceTestOverflow =
+        { surface_testOverflow, "surface_testOverflow", "Test overflow detection.", TEST_ENABLED};
+
 /* Sequence of Surface test cases */
 static const SDLTest_TestCaseReference *surfaceTests[] =  {
     &surfaceTest1, &surfaceTest2, &surfaceTest3, &surfaceTest4, &surfaceTest5,
     &surfaceTest6, &surfaceTest7, &surfaceTest8, &surfaceTest9, &surfaceTest10,
-    &surfaceTest11, &surfaceTest12, NULL
+    &surfaceTest11, &surfaceTest12, &surfaceTestOverflow, NULL
 };
 
 /* Surface test suite (global) */