From 4ffa7f868d830e808a40bb987a0d6b1264283eec Mon Sep 17 00:00:00 2001
From: Sylvain Becker <[EMAIL REDACTED]>
Date: Sat, 3 Dec 2022 21:22:14 +0100
Subject: [PATCH] Fixed bug libsdl-org#6745 - Check for overflow in
SDL_CalculateYUVSize (#6747)
Fixed bug #6745 - Check for overflow in SDL_CalculateYUVSize
---
src/render/SDL_yuv_sw.c | 5 +-
src/video/SDL_surface.c | 52 +++++++++++++------
src/video/SDL_yuv.c | 107 +++++++++++++++++++++++++++++++++++-----
3 files changed, 136 insertions(+), 28 deletions(-)
diff --git a/src/render/SDL_yuv_sw.c b/src/render/SDL_yuv_sw.c
index f1a37c4f0a5b..55d02ce00beb 100644
--- a/src/render/SDL_yuv_sw.c
+++ b/src/render/SDL_yuv_sw.c
@@ -58,7 +58,10 @@ SDL_SW_CreateYUVTexture(Uint32 format, int w, int h)
swdata->h = h;
{
size_t dst_size;
- SDL_CalculateYUVSize(format, w, h, &dst_size, NULL);
+ if (SDL_CalculateYUVSize(format, w, h, &dst_size, NULL) < 0) {
+ SDL_OutOfMemory();
+ return NULL;
+ }
swdata->pixels = (Uint8 *)SDL_SIMDAlloc(dst_size);
if (!swdata->pixels) {
SDL_SW_DestroyYUVTexture(swdata);
diff --git a/src/video/SDL_surface.c b/src/video/SDL_surface.c
index 0a25af374fe7..3255b73e7a12 100644
--- a/src/video/SDL_surface.c
+++ b/src/video/SDL_surface.c
@@ -39,13 +39,15 @@ SDL_COMPILE_TIME_ASSERT(can_indicate_overflow, SDL_SIZE_MAX > SDL_MAX_SINT32);
/*
* Calculate the pad-aligned scanline width of a surface.
* Return SDL_SIZE_MAX on overflow.
+ *
+ * for FOURCC, use SDL_CalculateYUVSize()
*/
static size_t
SDL_CalculatePitch(Uint32 format, size_t width, SDL_bool minimal)
{
size_t pitch;
- if (SDL_ISPIXELFORMAT_FOURCC(format) || SDL_BITSPERPIXEL(format) >= 8) {
+ if (SDL_BITSPERPIXEL(format) >= 8) {
if (SDL_size_mul_overflow(width, SDL_BYTESPERPIXEL(format), &pitch)) {
return SDL_SIZE_MAX;
}
@@ -88,11 +90,21 @@ SDL_CreateSurface(int width, int height, Uint32 format)
return NULL;
}
- pitch = SDL_CalculatePitch(format, width, SDL_FALSE);
- if (pitch > SDL_MAX_SINT32) {
- /* Overflow... */
- SDL_OutOfMemory();
- return NULL;
+ if (SDL_ISPIXELFORMAT_FOURCC(format)) {
+ int p;
+ if (SDL_CalculateYUVSize(format, width, height, NULL, &p) < 0) {
+ /* Overflow... */
+ SDL_OutOfMemory();
+ return NULL;
+ }
+ pitch = p;
+ } else {
+ pitch = SDL_CalculatePitch(format, width, SDL_FALSE);
+ if (pitch > SDL_MAX_SINT32) {
+ /* Overflow... */
+ SDL_OutOfMemory();
+ return NULL;
+ }
}
/* Allocate the surface */
@@ -134,20 +146,21 @@ SDL_CreateSurface(int width, int height, Uint32 format)
/* Get the pixels */
if (surface->w && surface->h) {
- /* Assumptions checked in surface_size_assumptions assert above */
size_t size;
- if (SDL_size_mul_overflow(surface->h, surface->pitch, &size)) {
- /* Overflow... */
- SDL_FreeSurface(surface);
- SDL_OutOfMemory();
- return NULL;
- }
-
if (SDL_ISPIXELFORMAT_FOURCC(surface->format->format)) {
/* Get correct size and pitch for YUV formats */
if (SDL_CalculateYUVSize(surface->format->format, surface->w, surface->h, &size, &surface->pitch) < 0) {
+ /* Overflow... */
+ SDL_FreeSurface(surface);
SDL_OutOfMemory();
+ return NULL;
+ }
+ } else {
+ /* Assumptions checked in surface_size_assumptions assert above */
+ if (SDL_size_mul_overflow(surface->h, surface->pitch, &size)) {
+ /* Overflow... */
SDL_FreeSurface(surface);
+ SDL_OutOfMemory();
return NULL;
}
}
@@ -202,7 +215,16 @@ SDL_CreateSurfaceFrom(void *pixels,
return NULL;
}
- minimalPitch = SDL_CalculatePitch(format, width, SDL_TRUE);
+ if (SDL_ISPIXELFORMAT_FOURCC(format)) {
+ int p;
+ if (SDL_CalculateYUVSize(format, width, height, NULL, &p) < 0) {
+ SDL_InvalidParamError("pitch");
+ return NULL;
+ }
+ minimalPitch = p;
+ } else {
+ minimalPitch = SDL_CalculatePitch(format, width, SDL_TRUE);
+ }
if (pitch < 0 || (pitch > 0 && ((size_t)pitch) < minimalPitch)) {
SDL_InvalidParamError("pitch");
diff --git a/src/video/SDL_yuv.c b/src/video/SDL_yuv.c
index 773a375daff2..aff90926c7e0 100644
--- a/src/video/SDL_yuv.c
+++ b/src/video/SDL_yuv.c
@@ -29,6 +29,10 @@
static SDL_YUV_CONVERSION_MODE SDL_YUV_ConversionMode = SDL_YUV_CONVERSION_BT601;
+#if SDL_HAVE_YUV
+static SDL_bool IsPlanar2x2Format(Uint32 format);
+#endif
+
void SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_MODE mode)
{
SDL_YUV_ConversionMode = mode;
@@ -53,51 +57,130 @@ SDL_YUV_CONVERSION_MODE SDL_GetYUVConversionModeForResolution(int width, int hei
}
/*
- * Calculate YUV size.
+ * Calculate YUV size and pitch. Check for overflow.
* Output 'pitch' that can be used with SDL_ConvertPixels()
*
* return 0 on success, -1 on error
*/
int SDL_CalculateYUVSize(Uint32 format, int w, int h, size_t *size, int *pitch)
{
- const int sz_plane = w * h;
- const int sz_plane_chroma = ((w + 1) / 2) * ((h + 1) / 2);
- const int sz_plane_packed = ((w + 1) / 2) * h;
- int dst_size = 0;
+#if SDL_HAVE_YUV
+ int sz_plane = 0, sz_plane_chroma = 0, sz_plane_packed = 0;
+
+ if (IsPlanar2x2Format(format) == SDL_TRUE) {
+ {
+ /* sz_plane == w * h; */
+ size_t s1;
+ if (SDL_size_mul_overflow(w, h, &s1) < 0) {
+ return -1;
+ }
+ sz_plane = (int) s1;
+ }
+
+ {
+ /* sz_plane_chroma == ((w + 1) / 2) * ((h + 1) / 2); */
+ size_t s1, s2, s3;
+ if (SDL_size_add_overflow(w, 1, &s1) < 0) {
+ return -1;
+ }
+ s1 = s1 / 2;
+ if (SDL_size_add_overflow(h, 1, &s2) < 0) {
+ return -1;
+ }
+ s2 = s2 / 2;
+ if (SDL_size_mul_overflow(s1, s2, &s3) < 0) {
+ return -1;
+ }
+ sz_plane_chroma = (int) s3;
+ }
+ } else {
+ /* sz_plane_packed == ((w + 1) / 2) * h; */
+ size_t s1, s2;
+ if (SDL_size_add_overflow(w, 1, &s1) < 0) {
+ return -1;
+ }
+ s1 = s1 / 2;
+ if (SDL_size_mul_overflow(s1, h, &s2) < 0) {
+ return -1;
+ }
+ sz_plane_packed = (int) s2;
+ }
+
switch (format) {
case SDL_PIXELFORMAT_YV12: /**< Planar mode: Y + V + U (3 planes) */
case SDL_PIXELFORMAT_IYUV: /**< Planar mode: Y + U + V (3 planes) */
- dst_size = sz_plane + sz_plane_chroma + sz_plane_chroma;
+
if (pitch) {
*pitch = w;
}
+
+ if (size) {
+ /* dst_size == sz_plane + sz_plane_chroma + sz_plane_chroma; */
+ size_t s1, s2;
+ if (SDL_size_add_overflow(sz_plane, sz_plane_chroma, &s1) < 0) {
+ return -1;
+ }
+ if (SDL_size_add_overflow(s1, sz_plane_chroma, &s2) < 0) {
+ return -1;
+ }
+ *size = (int)s2;
+ }
break;
case SDL_PIXELFORMAT_YUY2: /**< Packed mode: Y0+U0+Y1+V0 (1 plane) */
case SDL_PIXELFORMAT_UYVY: /**< Packed mode: U0+Y0+V0+Y1 (1 plane) */
case SDL_PIXELFORMAT_YVYU: /**< Packed mode: Y0+V0+Y1+U0 (1 plane) */
- dst_size = 4 * sz_plane_packed;
+
if (pitch) {
- *pitch = ((w + 1) / 2) * 4;
+ /* pitch == ((w + 1) / 2) * 4; */
+ size_t p1, p2;
+ if (SDL_size_add_overflow(w, 1, &p1) < 0) {
+ return -1;
+ }
+ p1 = p1 / 2;
+ if (SDL_size_mul_overflow(p1, 4, &p2) < 0) {
+ return -1;
+ }
+ *pitch = (int) p2;
+ }
+
+ if (size) {
+ /* dst_size == 4 * sz_plane_packed; */
+ size_t s1;
+ if (SDL_size_mul_overflow(sz_plane_packed, 4, &s1) < 0) {
+ return -1;
+ }
+ *size = (int) s1;
}
break;
case SDL_PIXELFORMAT_NV12: /**< Planar mode: Y + U/V interleaved (2 planes) */
case SDL_PIXELFORMAT_NV21: /**< Planar mode: Y + V/U interleaved (2 planes) */
- dst_size = sz_plane + sz_plane_chroma + sz_plane_chroma;
if (pitch) {
*pitch = w;
}
+
+ if (size) {
+ /* dst_size == sz_plane + sz_plane_chroma + sz_plane_chroma; */
+ size_t s1, s2;
+ if (SDL_size_add_overflow(sz_plane, sz_plane_chroma, &s1) < 0) {
+ return -1;
+ }
+ if (SDL_size_add_overflow(s1, sz_plane_chroma, &s2) < 0) {
+ return -1;
+ }
+ *size = (int) s2;
+ }
break;
default:
return -1;
}
- if (size) {
- *size = dst_size;
- }
return 0;
+#else
+ return -1;
+#endif
}
#if SDL_HAVE_YUV