From 41d38c0f64a4d7da0b2d1e9d2711c30bc136a180 Mon Sep 17 00:00:00 2001
From: "Ryan C. Gordon" <[EMAIL REDACTED]>
Date: Wed, 26 Oct 2022 09:43:04 -0400
Subject: [PATCH] shape: More robust handling of failure cases in CreateShaper.
---
src/video/cocoa/SDL_cocoashape.m | 8 +++++++-
src/video/directfb/SDL_DirectFB_shape.c | 9 +++++++++
src/video/windows/SDL_windowsshape.c | 12 ++++++++++--
src/video/x11/SDL_x11shape.c | 23 +++++++++++++++++------
4 files changed, 43 insertions(+), 9 deletions(-)
diff --git a/src/video/cocoa/SDL_cocoashape.m b/src/video/cocoa/SDL_cocoashape.m
index 6dbd64bd2b1b..ac3459c5613d 100644
--- a/src/video/cocoa/SDL_cocoashape.m
+++ b/src/video/cocoa/SDL_cocoashape.m
@@ -47,11 +47,17 @@ @implementation SDL_CocoaClosure
SDL_ShapeData* data;
int resized_properly;
SDL_WindowData* windata = (__bridge SDL_WindowData*)window->driverdata;
+
+ result = (SDL_WindowShaper *)SDL_malloc(sizeof(SDL_WindowShaper));
+ if (!result) {
+ SDL_OutOfMemory();
+ return NULL;
+ }
+
[windata.nswindow setOpaque:NO];
[windata.nswindow setStyleMask:NSWindowStyleMaskBorderless];
- result = (SDL_WindowShaper *)SDL_malloc(sizeof(SDL_WindowShaper));
result->window = window;
result->mode.mode = ShapeModeDefault;
result->mode.parameters.binarizationCutoff = 1;
diff --git a/src/video/directfb/SDL_DirectFB_shape.c b/src/video/directfb/SDL_DirectFB_shape.c
index ff334c80491d..b6f5d0811c00 100644
--- a/src/video/directfb/SDL_DirectFB_shape.c
+++ b/src/video/directfb/SDL_DirectFB_shape.c
@@ -35,11 +35,20 @@ DirectFB_CreateShaper(SDL_Window* window) {
int resized_properly;
result = SDL_malloc(sizeof(SDL_WindowShaper));
+ if (!result) {
+ SDL_OutOfMemory();
+ return NULL;
+ }
result->window = window;
result->mode.mode = ShapeModeDefault;
result->mode.parameters.binarizationCutoff = 1;
result->userx = result->usery = 0;
data = SDL_malloc(sizeof(SDL_ShapeData));
+ if (!data) {
+ SDL_free(result);
+ SDL_OutOfMemory();
+ return NULL;
+ }
result->driverdata = data;
data->surface = NULL;
window->shaper = result;
diff --git a/src/video/windows/SDL_windowsshape.c b/src/video/windows/SDL_windowsshape.c
index 919076f9a215..484566dd30d7 100644
--- a/src/video/windows/SDL_windowsshape.c
+++ b/src/video/windows/SDL_windowsshape.c
@@ -29,13 +29,21 @@ SDL_WindowShaper*
Win32_CreateShaper(SDL_Window * window) {
int resized_properly;
SDL_WindowShaper* result = (SDL_WindowShaper *)SDL_malloc(sizeof(SDL_WindowShaper));
+ if (!result) {
+ SDL_OutOfMemory();
+ return NULL;
+ }
result->window = window;
result->mode.mode = ShapeModeDefault;
result->mode.parameters.binarizationCutoff = 1;
result->userx = result->usery = 0;
result->hasshape = SDL_FALSE;
- result->driverdata = (SDL_ShapeData*)SDL_malloc(sizeof(SDL_ShapeData));
- ((SDL_ShapeData*)result->driverdata)->mask_tree = NULL;
+ result->driverdata = (SDL_ShapeData*)SDL_calloc(1, sizeof(SDL_ShapeData));
+ if (!result->driverdata) {
+ SDL_free(result);
+ SDL_OutOfMemory();
+ return NULL;
+ }
window->shaper = result;
/* Put some driver-data here. */
resized_properly = Win32_ResizeWindowShape(window);
diff --git a/src/video/x11/SDL_x11shape.c b/src/video/x11/SDL_x11shape.c
index 7d78f11274a3..7d4e32c22e91 100644
--- a/src/video/x11/SDL_x11shape.c
+++ b/src/video/x11/SDL_x11shape.c
@@ -31,22 +31,34 @@ SDL_WindowShaper*
X11_CreateShaper(SDL_Window* window) {
SDL_WindowShaper* result = NULL;
SDL_ShapeData* data = NULL;
- int resized_properly;
#if SDL_VIDEO_DRIVER_X11_XSHAPE
if (SDL_X11_HAVE_XSHAPE) { /* Make sure X server supports it. */
result = SDL_malloc(sizeof(SDL_WindowShaper));
+ if (!result) {
+ SDL_OutOfMemory();
+ return NULL;
+ }
result->window = window;
result->mode.mode = ShapeModeDefault;
result->mode.parameters.binarizationCutoff = 1;
result->userx = result->usery = 0;
data = SDL_malloc(sizeof(SDL_ShapeData));
+ if (!data) {
+ SDL_free(result);
+ SDL_OutOfMemory();
+ return NULL;
+ }
result->driverdata = data;
data->bitmapsize = 0;
data->bitmap = NULL;
window->shaper = result;
- resized_properly = X11_ResizeWindowShape(window);
- SDL_assert(resized_properly == 0);
+ if (X11_ResizeWindowShape(window) != 0) {
+ SDL_free(result);
+ SDL_free(data);
+ window->shaper = NULL;
+ return NULL;
+ }
}
#endif
@@ -64,11 +76,10 @@ X11_ResizeWindowShape(SDL_Window* window) {
bitmapsize *= window->h;
if(data->bitmapsize != bitmapsize || data->bitmap == NULL) {
data->bitmapsize = bitmapsize;
- if(data->bitmap != NULL)
- SDL_free(data->bitmap);
+ SDL_free(data->bitmap);
data->bitmap = SDL_malloc(data->bitmapsize);
if(data->bitmap == NULL) {
- return SDL_SetError("Could not allocate memory for shaped-window bitmap.");
+ return SDL_OutOfMemory();
}
}
SDL_memset(data->bitmap,0,data->bitmapsize);