SDL: Fixed crashes handling D3D11/12 device lost in testsprite

From 315842cf71997b2e25b12d7127eeeb94ded7fea1 Mon Sep 17 00:00:00 2001
From: Sam Lantinga <[EMAIL REDACTED]>
Date: Mon, 21 Oct 2024 00:12:16 -0700
Subject: [PATCH] Fixed crashes handling D3D11/12 device lost in testsprite

You can test this using "dxcap -forcetdr"
---
 src/render/direct3d11/SDL_render_d3d11.c | 19 +++++++++++++++++--
 src/render/direct3d12/SDL_render_d3d12.c | 12 +++++++++++-
 test/testsprite.c                        |  8 +++++++-
 3 files changed, 35 insertions(+), 4 deletions(-)

diff --git a/src/render/direct3d11/SDL_render_d3d11.c b/src/render/direct3d11/SDL_render_d3d11.c
index 1d9b291346f11..f75bf02e3c7f5 100644
--- a/src/render/direct3d11/SDL_render_d3d11.c
+++ b/src/render/direct3d11/SDL_render_d3d11.c
@@ -336,7 +336,7 @@ static void D3D11_ReleaseAll(SDL_Renderer *renderer)
                 SAFE_RELEASE(data->blendModes[i].blendState);
             }
             SDL_free(data->blendModes);
-
+            data->blendModes = NULL;
             data->blendModesCount = 0;
         }
         for (i = 0; i < SDL_arraysize(data->pixelShaders); ++i) {
@@ -993,12 +993,14 @@ static HRESULT D3D11_HandleDeviceLost(SDL_Renderer *renderer)
     result = D3D11_CreateDeviceResources(renderer);
     if (FAILED(result)) {
         // D3D11_CreateDeviceResources will set the SDL error
+        D3D11_ReleaseAll(renderer);
         return result;
     }
 
     result = D3D11_UpdateForWindowSizeChange(renderer);
     if (FAILED(result)) {
         // D3D11_UpdateForWindowSizeChange will set the SDL error
+        D3D11_ReleaseAll(renderer);
         return result;
     }
 
@@ -2395,6 +2397,10 @@ static bool D3D11_RunCommandQueue(SDL_Renderer *renderer, SDL_RenderCommand *cmd
     D3D11_RenderData *rendererData = (D3D11_RenderData *)renderer->internal;
     const int viewportRotation = D3D11_GetRotationForCurrentRenderTarget(renderer);
 
+    if (!rendererData->d3dDevice) {
+        return SDL_SetError("Device lost and couldn't be recovered");
+    }
+
     if (rendererData->pixelSizeChanged) {
         D3D11_UpdateForWindowSizeChange(renderer);
         rendererData->pixelSizeChanged = false;
@@ -2613,6 +2619,10 @@ static bool D3D11_RenderPresent(SDL_Renderer *renderer)
     HRESULT result;
     DXGI_PRESENT_PARAMETERS parameters;
 
+    if (!data->d3dDevice) {
+        return SDL_SetError("Device lost and couldn't be recovered");
+    }
+
     SDL_zero(parameters);
 
     /* The application may optionally specify "dirty" or "scroll"
@@ -2634,10 +2644,15 @@ static bool D3D11_RenderPresent(SDL_Renderer *renderer)
          * must recreate all device resources.
          */
         if (result == DXGI_ERROR_DEVICE_REMOVED) {
-            D3D11_HandleDeviceLost(renderer);
+            if (SUCCEEDED(D3D11_HandleDeviceLost(renderer))) {
+                SDL_SetError("Present failed, device lost");
+            } else {
+                // Recovering from device lost failed, error is already set
+            }
         } else if (result == DXGI_ERROR_INVALID_CALL) {
             // We probably went through a fullscreen <-> windowed transition
             D3D11_CreateWindowSizeDependentResources(renderer);
+            WIN_SetErrorFromHRESULT(SDL_COMPOSE_ERROR("IDXGISwapChain::Present"), result);
         } else {
             WIN_SetErrorFromHRESULT(SDL_COMPOSE_ERROR("IDXGISwapChain::Present"), result);
         }
diff --git a/src/render/direct3d12/SDL_render_d3d12.c b/src/render/direct3d12/SDL_render_d3d12.c
index df6e7e417e06a..ec3cfe9dbbbfa 100644
--- a/src/render/direct3d12/SDL_render_d3d12.c
+++ b/src/render/direct3d12/SDL_render_d3d12.c
@@ -414,6 +414,7 @@ static void D3D12_ReleaseAll(SDL_Renderer *renderer)
                 D3D_SAFE_RELEASE(data->pipelineStates[i].pipelineState);
             }
             SDL_free(data->pipelineStates);
+            data->pipelineStates = NULL;
             data->pipelineStateCount = 0;
         }
 
@@ -2738,6 +2739,10 @@ static bool D3D12_SetCopyState(SDL_Renderer *renderer, const SDL_RenderCommand *
     D3D12_CPU_DESCRIPTOR_HANDLE *textureSampler;
     D3D12_PixelShaderConstants constants;
 
+    if (!textureData) {
+        return SDL_SetError("Texture is not currently available");
+    }
+
     D3D12_SetupShaderConstants(renderer, cmd, texture, &constants);
 
     switch (textureData->scaleMode) {
@@ -3132,10 +3137,15 @@ static bool D3D12_RenderPresent(SDL_Renderer *renderer)
          * must recreate all device resources.
          */
         if (result == DXGI_ERROR_DEVICE_REMOVED) {
-            D3D12_HandleDeviceLost(renderer);
+            if (SUCCEEDED(D3D12_HandleDeviceLost(renderer))) {
+                SDL_SetError("Present failed, device lost");
+            } else {
+                // Recovering from device lost failed, error is already set
+            }
         } else if (result == DXGI_ERROR_INVALID_CALL) {
             // We probably went through a fullscreen <-> windowed transition
             D3D12_CreateWindowSizeDependentResources(renderer);
+            WIN_SetErrorFromHRESULT(SDL_COMPOSE_ERROR("IDXGISwapChain::Present"), result);
         } else {
             WIN_SetErrorFromHRESULT(SDL_COMPOSE_ERROR("IDXGISwapChain::Present"), result);
         }
diff --git a/test/testsprite.c b/test/testsprite.c
index 1c9fdddb0bd76..fa04bfcfeedf2 100644
--- a/test/testsprite.c
+++ b/test/testsprite.c
@@ -21,6 +21,7 @@
 #define MAX_SPEED   1
 
 static SDLTest_CommonState *state;
+static const char *icon = "icon.bmp";
 static int num_sprites;
 static SDL_Texture **sprites;
 static bool cycle_color;
@@ -56,6 +57,9 @@ static int LoadSprite(const char *file)
 
     for (i = 0; i < state->num_windows; ++i) {
         /* This does the SDL_LoadBMP step repeatedly, but that's OK for test code. */
+        if (sprites[i]) {
+            SDL_DestroyTexture(sprites[i]);
+        }
         sprites[i] = LoadTexture(state->renderers[i], file, true, &w, &h);
         sprite_w = (float)w;
         sprite_h = (float)h;
@@ -390,7 +394,6 @@ SDL_AppResult SDL_AppInit(void **appstate, int argc, char *argv[])
     SDL_Rect safe_area;
     int i;
     Uint64 seed;
-    const char *icon = "icon.bmp";
 
     /* Initialize parameters */
     num_sprites = NUM_SPRITES;
@@ -553,6 +556,9 @@ SDL_AppResult SDL_AppInit(void **appstate, int argc, char *argv[])
 
 SDL_AppResult SDL_AppEvent(void *appstate, SDL_Event *event)
 {
+    if (event->type == SDL_EVENT_RENDER_DEVICE_RESET) {
+        LoadSprite(icon);
+    }
     return SDLTest_CommonEventMainCallbacks(state, event);
 }