SDL: Added a hint for alternate OpenGL NV12 data format

From b360965d0d349fc977bd73ba70c2e8c2c23544b3 Mon Sep 17 00:00:00 2001
From: Sam Lantinga <[EMAIL REDACTED]>
Date: Thu, 21 Oct 2021 20:48:05 -0700
Subject: [PATCH] Added a hint for alternate OpenGL NV12 data format

---
 src/render/opengl/SDL_render_gl.c  | 100 +++++++++++++++--------------
 src/render/opengl/SDL_shaders_gl.c |  55 ++++++++++++++--
 src/render/opengl/SDL_shaders_gl.h |   6 +-
 3 files changed, 106 insertions(+), 55 deletions(-)

diff --git a/src/render/opengl/SDL_render_gl.c b/src/render/opengl/SDL_render_gl.c
index 80cfd09c10..6d68bc723a 100644
--- a/src/render/opengl/SDL_render_gl.c
+++ b/src/render/opengl/SDL_render_gl.c
@@ -123,6 +123,7 @@ typedef struct
     GLfloat texh;
     GLenum format;
     GLenum formattype;
+    GL_Shader shader;
     void *pixels;
     int pitch;
     SDL_Rect locked_rect;
@@ -630,6 +631,57 @@ GL_CreateTexture(SDL_Renderer * renderer, SDL_Texture * texture)
     }
 #endif
 
+    if (texture->format == SDL_PIXELFORMAT_ABGR8888 || texture->format == SDL_PIXELFORMAT_ARGB8888) {
+        data->shader = SHADER_RGBA;
+    } else {
+        data->shader = SHADER_RGB;
+    }
+
+#if SDL_HAVE_YUV
+    if (data->yuv || data->nv12) {
+        switch (SDL_GetYUVConversionModeForResolution(texture->w, texture->h)) {
+            case SDL_YUV_CONVERSION_JPEG:
+                if (data->yuv) {
+                    data->shader = SHADER_YUV_JPEG;
+                } else if (texture->format == SDL_PIXELFORMAT_NV12) {
+                    data->shader = SHADER_NV12_JPEG;
+                } else {
+                    data->shader = SHADER_NV21_JPEG;
+                }
+                break;
+            case SDL_YUV_CONVERSION_BT601:
+                if (data->yuv) {
+                    data->shader = SHADER_YUV_BT601;
+                } else if (texture->format == SDL_PIXELFORMAT_NV12) {
+                    if (SDL_GetHintBoolean("SDL_RENDER_OPENGL_NV12_RG_SHADER", SDL_FALSE)) {
+                        data->shader = SHADER_NV12_RG_BT601;
+                    } else {
+                        data->shader = SHADER_NV12_RA_BT601;
+                    }
+                } else {
+                    data->shader = SHADER_NV21_BT601;
+                }
+                break;
+            case SDL_YUV_CONVERSION_BT709:
+                if (data->yuv) {
+                    data->shader = SHADER_YUV_BT709;
+                } else if (texture->format == SDL_PIXELFORMAT_NV12) {
+                    if (SDL_GetHintBoolean("SDL_RENDER_OPENGL_NV12_RG_SHADER", SDL_FALSE)) {
+                        data->shader = SHADER_NV12_RG_BT709;
+                    } else {
+                        data->shader = SHADER_NV12_RA_BT709;
+                    }
+                } else {
+                    data->shader = SHADER_NV21_BT709;
+                }
+                break;
+            default:
+                SDL_assert(!"unsupported YUV conversion mode");
+                break;
+        }
+    }
+#endif /* SDL_HAVE_YUV */
+
     return GL_CheckError("", renderer);
 }
 
@@ -1186,54 +1238,8 @@ SetCopyState(GL_RenderData *data, const SDL_RenderCommand *cmd)
 {
     SDL_Texture *texture = cmd->data.draw.texture;
     const GL_TextureData *texturedata = (GL_TextureData *) texture->driverdata;
-    GL_Shader shader;
-
-    if (texture->format == SDL_PIXELFORMAT_ABGR8888 || texture->format == SDL_PIXELFORMAT_ARGB8888) {
-        shader = SHADER_RGBA;
-    } else {
-        shader = SHADER_RGB;
-    }
-
-#if SDL_HAVE_YUV
-    if (data->shaders) {
-        if (texturedata->yuv || texturedata->nv12) {
-            switch (SDL_GetYUVConversionModeForResolution(texture->w, texture->h)) {
-                case SDL_YUV_CONVERSION_JPEG:
-                    if (texturedata->yuv) {
-                        shader = SHADER_YUV_JPEG;
-                    } else if (texture->format == SDL_PIXELFORMAT_NV12) {
-                        shader = SHADER_NV12_JPEG;
-                    } else {
-                        shader = SHADER_NV21_JPEG;
-                    }
-                    break;
-                case SDL_YUV_CONVERSION_BT601:
-                    if (texturedata->yuv) {
-                        shader = SHADER_YUV_BT601;
-                    } else if (texture->format == SDL_PIXELFORMAT_NV12) {
-                        shader = SHADER_NV12_BT601;
-                    } else {
-                        shader = SHADER_NV21_BT601;
-                    }
-                    break;
-                case SDL_YUV_CONVERSION_BT709:
-                    if (texturedata->yuv) {
-                        shader = SHADER_YUV_BT709;
-                    } else if (texture->format == SDL_PIXELFORMAT_NV12) {
-                        shader = SHADER_NV12_BT709;
-                    } else {
-                        shader = SHADER_NV21_BT709;
-                    }
-                    break;
-                default:
-                    SDL_assert(!"unsupported YUV conversion mode");
-                    break;
-            }
-        }
-    }
-#endif
 
-    SetDrawState(data, cmd, shader);
+    SetDrawState(data, cmd, texturedata->shader);
 
     if (texture != data->drawstate.texture) {
         const GLenum textype = data->textype;
diff --git a/src/render/opengl/SDL_shaders_gl.c b/src/render/opengl/SDL_shaders_gl.c
index 23f708acc3..5b6aea4633 100644
--- a/src/render/opengl/SDL_shaders_gl.c
+++ b/src/render/opengl/SDL_shaders_gl.c
@@ -149,7 +149,7 @@ struct GL_ShaderContext
 "uniform sampler2D tex1; // U/V \n"                             \
 "\n"                                                            \
 
-#define NV12_SHADER_BODY                                        \
+#define NV12_RA_SHADER_BODY                                     \
 "\n"                                                            \
 "void main()\n"                                                 \
 "{\n"                                                           \
@@ -174,6 +174,31 @@ struct GL_ShaderContext
 "    gl_FragColor = vec4(rgb, 1.0) * v_color;\n"                \
 "}"                                                             \
 
+#define NV12_RG_SHADER_BODY                                     \
+"\n"                                                            \
+"void main()\n"                                                 \
+"{\n"                                                           \
+"    vec2 tcoord;\n"                                            \
+"    vec3 yuv, rgb;\n"                                          \
+"\n"                                                            \
+"    // Get the Y value \n"                                     \
+"    tcoord = v_texCoord;\n"                                    \
+"    yuv.x = texture2D(tex0, tcoord).r;\n"                      \
+"\n"                                                            \
+"    // Get the U and V values \n"                              \
+"    tcoord *= UVCoordScale;\n"                                 \
+"    yuv.yz = texture2D(tex1, tcoord).rg;\n"                    \
+"\n"                                                            \
+"    // Do the color transform \n"                              \
+"    yuv += offset;\n"                                          \
+"    rgb.r = dot(yuv, Rcoeff);\n"                               \
+"    rgb.g = dot(yuv, Gcoeff);\n"                               \
+"    rgb.b = dot(yuv, Bcoeff);\n"                               \
+"\n"                                                            \
+"    // That was easy. :) \n"                                   \
+"    gl_FragColor = vec4(rgb, 1.0) * v_color;\n"                \
+"}"                                                             \
+
 #define NV21_SHADER_PROLOGUE                                    \
 "varying vec4 v_color;\n"                                       \
 "varying vec2 v_texCoord;\n"                                    \
@@ -294,25 +319,43 @@ static const char *shader_source[NUM_SHADERS][2] =
         /* fragment shader */
         NV12_SHADER_PROLOGUE
         JPEG_SHADER_CONSTANTS
-        NV12_SHADER_BODY
+        NV12_RA_SHADER_BODY
+    },
+    /* SHADER_NV12_RA_BT601 */
+    {
+        /* vertex shader */
+        TEXTURE_VERTEX_SHADER,
+        /* fragment shader */
+        NV12_SHADER_PROLOGUE
+        BT601_SHADER_CONSTANTS
+        NV12_RA_SHADER_BODY
     },
-    /* SHADER_NV12_BT601 */
+    /* SHADER_NV12_RG_BT601 */
     {
         /* vertex shader */
         TEXTURE_VERTEX_SHADER,
         /* fragment shader */
         NV12_SHADER_PROLOGUE
         BT601_SHADER_CONSTANTS
-        NV12_SHADER_BODY
+        NV12_RG_SHADER_BODY
+    },
+    /* SHADER_NV12_RA_BT709 */
+    {
+        /* vertex shader */
+        TEXTURE_VERTEX_SHADER,
+        /* fragment shader */
+        NV12_SHADER_PROLOGUE
+        BT709_SHADER_CONSTANTS
+        NV12_RA_SHADER_BODY
     },
-    /* SHADER_NV12_BT709 */
+    /* SHADER_NV12_RG_BT709 */
     {
         /* vertex shader */
         TEXTURE_VERTEX_SHADER,
         /* fragment shader */
         NV12_SHADER_PROLOGUE
         BT709_SHADER_CONSTANTS
-        NV12_SHADER_BODY
+        NV12_RG_SHADER_BODY
     },
     /* SHADER_NV21_JPEG */
     {
diff --git a/src/render/opengl/SDL_shaders_gl.h b/src/render/opengl/SDL_shaders_gl.h
index ece3ef87c2..0ccc13c992 100644
--- a/src/render/opengl/SDL_shaders_gl.h
+++ b/src/render/opengl/SDL_shaders_gl.h
@@ -36,8 +36,10 @@ typedef enum {
     SHADER_YUV_BT601,
     SHADER_YUV_BT709,
     SHADER_NV12_JPEG,
-    SHADER_NV12_BT601,
-    SHADER_NV12_BT709,
+    SHADER_NV12_RA_BT601,
+    SHADER_NV12_RG_BT601,
+    SHADER_NV12_RA_BT709,
+    SHADER_NV12_RG_BT709,
     SHADER_NV21_JPEG,
     SHADER_NV21_BT601,
     SHADER_NV21_BT709,