From d234f7a49874662e4c2c34d0ed8ae0f8bd90183e Mon Sep 17 00:00:00 2001
From: Sylvain Becker <[EMAIL REDACTED]>
Date: Mon, 2 Jan 2023 18:09:57 +0100
Subject: [PATCH] SDL2: backport SDL_CalculateYUVSize() that checks for YUV
size overflow (#6972)
---
src/render/SDL_yuv_sw.c | 29 ++-------
src/video/SDL_surface.c | 24 ++++++--
src/video/SDL_yuv.c | 131 ++++++++++++++++++++++++++++++++++++++++
src/video/SDL_yuv_c.h | 3 +
4 files changed, 157 insertions(+), 30 deletions(-)
diff --git a/src/render/SDL_yuv_sw.c b/src/render/SDL_yuv_sw.c
index ced9d4b53c96..321ac2e460f7 100644
--- a/src/render/SDL_yuv_sw.c
+++ b/src/render/SDL_yuv_sw.c
@@ -57,30 +57,11 @@ SDL_SW_CreateYUVTexture(Uint32 format, int w, int h)
swdata->w = w;
swdata->h = h;
{
- const int sz_plane = w * h;
- const int sz_plane_chroma = ((w + 1) / 2) * ((h + 1) / 2);
- const int sz_plane_packed = ((w + 1) / 2) * h;
- int dst_size = 0;
- switch (format) {
- case SDL_PIXELFORMAT_YV12: /**< Planar mode: Y + V + U (3 planes) */
- case SDL_PIXELFORMAT_IYUV: /**< Planar mode: Y + U + V (3 planes) */
- dst_size = sz_plane + sz_plane_chroma + sz_plane_chroma;
- break;
-
- case SDL_PIXELFORMAT_YUY2: /**< Packed mode: Y0+U0+Y1+V0 (1 plane) */
- case SDL_PIXELFORMAT_UYVY: /**< Packed mode: U0+Y0+V0+Y1 (1 plane) */
- case SDL_PIXELFORMAT_YVYU: /**< Packed mode: Y0+V0+Y1+U0 (1 plane) */
- dst_size = 4 * sz_plane_packed;
- break;
-
- case SDL_PIXELFORMAT_NV12: /**< Planar mode: Y + U/V interleaved (2 planes) */
- case SDL_PIXELFORMAT_NV21: /**< Planar mode: Y + V/U interleaved (2 planes) */
- dst_size = sz_plane + sz_plane_chroma + sz_plane_chroma;
- break;
-
- default:
- SDL_assert(0 && "We should never get here (caught above)");
- break;
+ size_t dst_size;
+ if (SDL_CalculateYUVSize(format, w, h, &dst_size, NULL) < 0) {
+ SDL_SW_DestroyYUVTexture(swdata);
+ SDL_OutOfMemory();
+ return NULL;
}
swdata->pixels = (Uint8 *)SDL_SIMDAlloc(dst_size);
if (!swdata->pixels) {
diff --git a/src/video/SDL_surface.c b/src/video/SDL_surface.c
index 99bd37b67b94..594dad413591 100644
--- a/src/video/SDL_surface.c
+++ b/src/video/SDL_surface.c
@@ -39,13 +39,15 @@ SDL_COMPILE_TIME_ASSERT(can_indicate_overflow, SDL_SIZE_MAX > SDL_MAX_SINT32);
/*
* Calculate the pad-aligned scanline width of a surface.
* Return SDL_SIZE_MAX on overflow.
+ *
+ * for FOURCC, use SDL_CalculateYUVSize()
*/
static size_t
SDL_CalculatePitch(Uint32 format, size_t width, SDL_bool minimal)
{
size_t pitch;
- if (SDL_ISPIXELFORMAT_FOURCC(format) || SDL_BITSPERPIXEL(format) >= 8) {
+ if (SDL_BITSPERPIXEL(format) >= 8) {
if (SDL_size_mul_overflow(width, SDL_BYTESPERPIXEL(format), &pitch)) {
return SDL_SIZE_MAX;
}
@@ -93,11 +95,16 @@ SDL_CreateRGBSurfaceWithFormat(Uint32 flags, int width, int height, int depth,
return NULL;
}
- pitch = SDL_CalculatePitch(format, width, SDL_FALSE);
- if (pitch > SDL_MAX_SINT32) {
- /* Overflow... */
- SDL_OutOfMemory();
+ if (SDL_ISPIXELFORMAT_FOURCC(format)) {
+ SDL_SetError("invalid format");
return NULL;
+ } else {
+ pitch = SDL_CalculatePitch(format, width, SDL_FALSE);
+ if (pitch > SDL_MAX_SINT32) {
+ /* Overflow... */
+ SDL_OutOfMemory();
+ return NULL;
+ }
}
/* Allocate the surface */
@@ -269,7 +276,12 @@ SDL_CreateRGBSurfaceWithFormatFrom(void *pixels,
return NULL;
}
- minimalPitch = SDL_CalculatePitch(format, width, SDL_TRUE);
+ if (SDL_ISPIXELFORMAT_FOURCC(format)) {
+ SDL_SetError("invalid format");
+ return NULL;
+ } else {
+ minimalPitch = SDL_CalculatePitch(format, width, SDL_TRUE);
+ }
if (pitch < 0 || (pitch > 0 && ((size_t)pitch) < minimalPitch)) {
SDL_InvalidParamError("pitch");
diff --git a/src/video/SDL_yuv.c b/src/video/SDL_yuv.c
index 897d5664267c..16805116a728 100644
--- a/src/video/SDL_yuv.c
+++ b/src/video/SDL_yuv.c
@@ -31,6 +31,10 @@
static SDL_YUV_CONVERSION_MODE SDL_YUV_ConversionMode = SDL_YUV_CONVERSION_BT601;
+#if SDL_HAVE_YUV
+static SDL_bool IsPlanar2x2Format(Uint32 format);
+#endif
+
void SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_MODE mode)
{
SDL_YUV_ConversionMode = mode;
@@ -54,6 +58,133 @@ SDL_YUV_CONVERSION_MODE SDL_GetYUVConversionModeForResolution(int width, int hei
return mode;
}
+/*
+ * Calculate YUV size and pitch. Check for overflow.
+ * Output 'pitch' that can be used with SDL_ConvertPixels()
+ *
+ * return 0 on success, -1 on error
+ */
+int SDL_CalculateYUVSize(Uint32 format, int w, int h, size_t *size, int *pitch)
+{
+#if SDL_HAVE_YUV
+ int sz_plane = 0, sz_plane_chroma = 0, sz_plane_packed = 0;
+
+ if (IsPlanar2x2Format(format) == SDL_TRUE) {
+ {
+ /* sz_plane == w * h; */
+ size_t s1;
+ if (SDL_size_mul_overflow(w, h, &s1) < 0) {
+ return -1;
+ }
+ sz_plane = (int) s1;
+ }
+
+ {
+ /* sz_plane_chroma == ((w + 1) / 2) * ((h + 1) / 2); */
+ size_t s1, s2, s3;
+ if (SDL_size_add_overflow(w, 1, &s1) < 0) {
+ return -1;
+ }
+ s1 = s1 / 2;
+ if (SDL_size_add_overflow(h, 1, &s2) < 0) {
+ return -1;
+ }
+ s2 = s2 / 2;
+ if (SDL_size_mul_overflow(s1, s2, &s3) < 0) {
+ return -1;
+ }
+ sz_plane_chroma = (int) s3;
+ }
+ } else {
+ /* sz_plane_packed == ((w + 1) / 2) * h; */
+ size_t s1, s2;
+ if (SDL_size_add_overflow(w, 1, &s1) < 0) {
+ return -1;
+ }
+ s1 = s1 / 2;
+ if (SDL_size_mul_overflow(s1, h, &s2) < 0) {
+ return -1;
+ }
+ sz_plane_packed = (int) s2;
+ }
+
+ switch (format) {
+ case SDL_PIXELFORMAT_YV12: /**< Planar mode: Y + V + U (3 planes) */
+ case SDL_PIXELFORMAT_IYUV: /**< Planar mode: Y + U + V (3 planes) */
+
+ if (pitch) {
+ *pitch = w;
+ }
+
+ if (size) {
+ /* dst_size == sz_plane + sz_plane_chroma + sz_plane_chroma; */
+ size_t s1, s2;
+ if (SDL_size_add_overflow(sz_plane, sz_plane_chroma, &s1) < 0) {
+ return -1;
+ }
+ if (SDL_size_add_overflow(s1, sz_plane_chroma, &s2) < 0) {
+ return -1;
+ }
+ *size = (int)s2;
+ }
+ break;
+
+ case SDL_PIXELFORMAT_YUY2: /**< Packed mode: Y0+U0+Y1+V0 (1 plane) */
+ case SDL_PIXELFORMAT_UYVY: /**< Packed mode: U0+Y0+V0+Y1 (1 plane) */
+ case SDL_PIXELFORMAT_YVYU: /**< Packed mode: Y0+V0+Y1+U0 (1 plane) */
+
+ if (pitch) {
+ /* pitch == ((w + 1) / 2) * 4; */
+ size_t p1, p2;
+ if (SDL_size_add_overflow(w, 1, &p1) < 0) {
+ return -1;
+ }
+ p1 = p1 / 2;
+ if (SDL_size_mul_overflow(p1, 4, &p2) < 0) {
+ return -1;
+ }
+ *pitch = (int) p2;
+ }
+
+ if (size) {
+ /* dst_size == 4 * sz_plane_packed; */
+ size_t s1;
+ if (SDL_size_mul_overflow(sz_plane_packed, 4, &s1) < 0) {
+ return -1;
+ }
+ *size = (int) s1;
+ }
+ break;
+
+ case SDL_PIXELFORMAT_NV12: /**< Planar mode: Y + U/V interleaved (2 planes) */
+ case SDL_PIXELFORMAT_NV21: /**< Planar mode: Y + V/U interleaved (2 planes) */
+ if (pitch) {
+ *pitch = w;
+ }
+
+ if (size) {
+ /* dst_size == sz_plane + sz_plane_chroma + sz_plane_chroma; */
+ size_t s1, s2;
+ if (SDL_size_add_overflow(sz_plane, sz_plane_chroma, &s1) < 0) {
+ return -1;
+ }
+ if (SDL_size_add_overflow(s1, sz_plane_chroma, &s2) < 0) {
+ return -1;
+ }
+ *size = (int) s2;
+ }
+ break;
+
+ default:
+ return -1;
+ }
+
+ return 0;
+#else
+ return -1;
+#endif
+}
+
#if SDL_HAVE_YUV
static int GetYUVConversionType(int width, int height, YCbCrType *yuv_type)
diff --git a/src/video/SDL_yuv_c.h b/src/video/SDL_yuv_c.h
index 29b56aeed42a..b2a2821c49b9 100644
--- a/src/video/SDL_yuv_c.h
+++ b/src/video/SDL_yuv_c.h
@@ -30,6 +30,9 @@ extern int SDL_ConvertPixels_YUV_to_RGB(int width, int height, Uint32 src_format
extern int SDL_ConvertPixels_RGB_to_YUV(int width, int height, Uint32 src_format, const void *src, int src_pitch, Uint32 dst_format, void *dst, int dst_pitch);
extern int SDL_ConvertPixels_YUV_to_YUV(int width, int height, Uint32 src_format, const void *src, int src_pitch, Uint32 dst_format, void *dst, int dst_pitch);
+
+extern int SDL_CalculateYUVSize(Uint32 format, int w, int h, size_t *size, int *pitch);
+
#endif /* SDL_yuv_c_h_ */
/* vi: set ts=4 sw=4 expandtab: */