sdl12-compat: Support Multisampling with OpenGL Logical Scaling

From dd88f13e5fcd4b36a7edc3530298682089c40046 Mon Sep 17 00:00:00 2001
From: David Gow <[EMAIL REDACTED]>
Date: Thu, 20 May 2021 18:02:08 +0800
Subject: [PATCH] Support Multisampling with OpenGL Logical Scaling

OpenGL Logical Scaling didn't work when Multisampling was enabled, as
the faux-backbuffer didn't have multiple samples, so the format didn't
match, making the blit fail and resulting in a black screen.

This change intercepts calls to set up MSAA in SDL_GL_SetAttribute(),
and, if OpenGL Logical Scaling is requested, creates an FBO with the
desired number of samples, and a second FBO used for multisample resolve
before scaling. (It's not possible for glBlitFramebuffer() to scale and
resolve at the same time -- at least without the
GL_EXT_framebuffer_multisample_blit_scaled extension.)

If OpenGL Logical Scaling has not been requested, we quickly
SDL_GL_SetAttribute() the appropriate values for real before creating
the window. This implementation does have the bug that MSAA won't work
if OpenGL Logical Scaling is requested, but fails to initialise, as
we've already created the window without it by the time we know we need
to fall back.
---
 src/SDL12_compat.c | 79 +++++++++++++++++++++++++++++++++++++++++++---
 src/SDL20_syms.h   |  2 ++
 2 files changed, 76 insertions(+), 5 deletions(-)

diff --git a/src/SDL12_compat.c b/src/SDL12_compat.c
index 9c21592..45da6b0 100644
--- a/src/SDL12_compat.c
+++ b/src/SDL12_compat.c
@@ -820,6 +820,10 @@ static int OpenGLLogicalScalingHeight = 0;
 static GLuint OpenGLLogicalScalingFBO = 0;
 static GLuint OpenGLLogicalScalingColor = 0;
 static GLuint OpenGLLogicalScalingDepth = 0;
+static int OpenGLLogicalScalingSamples = 0;
+static GLuint OpenGLLogicalScalingMultisampleFBO = 0;
+static GLuint OpenGLLogicalScalingMultisampleColor = 0;
+static GLuint OpenGLLogicalScalingMultisampleDepth = 0;
 
 
 /* !!! FIXME: need a mutex for the event queue. */
@@ -3330,11 +3334,11 @@ InitializeOpenGLScaling(const int w, const int h)
     OpenGLFuncs.glBindFramebuffer(GL_FRAMEBUFFER, OpenGLLogicalScalingFBO);
     OpenGLFuncs.glGenRenderbuffers(1, &OpenGLLogicalScalingColor);
     OpenGLFuncs.glBindRenderbuffer(GL_RENDERBUFFER, OpenGLLogicalScalingColor);
-    OpenGLFuncs.glRenderbufferStorage(GL_RENDERBUFFER, GL_RGB8, w, h);
+    OpenGLFuncs.glRenderbufferStorageMultisample(GL_RENDERBUFFER, OpenGLLogicalScalingSamples, GL_RGB8, w, h);
     OpenGLFuncs.glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, OpenGLLogicalScalingColor);
     OpenGLFuncs.glGenRenderbuffers(1, &OpenGLLogicalScalingDepth);
     OpenGLFuncs.glBindRenderbuffer(GL_RENDERBUFFER, OpenGLLogicalScalingDepth);
-    OpenGLFuncs.glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8, w, h);  /* !!! FIXME: is an extension (or core 3.0) */
+    OpenGLFuncs.glRenderbufferStorageMultisample(GL_RENDERBUFFER, OpenGLLogicalScalingSamples, GL_DEPTH24_STENCIL8, w, h);
     OpenGLFuncs.glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_RENDERBUFFER, OpenGLLogicalScalingDepth);
     OpenGLFuncs.glBindRenderbuffer(GL_RENDERBUFFER, 0);
 
@@ -3347,10 +3351,35 @@ InitializeOpenGLScaling(const int w, const int h)
         return SDL_FALSE;
     }
 
+    if (OpenGLLogicalScalingSamples) {
+        OpenGLFuncs.glGenFramebuffers(1, &OpenGLLogicalScalingMultisampleFBO);
+        OpenGLFuncs.glBindFramebuffer(GL_FRAMEBUFFER, OpenGLLogicalScalingMultisampleFBO);
+        OpenGLFuncs.glGenRenderbuffers(1, &OpenGLLogicalScalingMultisampleColor);
+        OpenGLFuncs.glBindRenderbuffer(GL_RENDERBUFFER, OpenGLLogicalScalingMultisampleColor);
+        OpenGLFuncs.glRenderbufferStorage(GL_RENDERBUFFER, GL_RGB8, w, h);
+        OpenGLFuncs.glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, OpenGLLogicalScalingMultisampleColor);
+        OpenGLFuncs.glGenRenderbuffers(1, &OpenGLLogicalScalingMultisampleDepth);
+        OpenGLFuncs.glBindRenderbuffer(GL_RENDERBUFFER, OpenGLLogicalScalingMultisampleDepth);
+        OpenGLFuncs.glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8, w, h);  /* !!! FIXME: is an extension (or core 3.0) */
+        OpenGLFuncs.glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_RENDERBUFFER, OpenGLLogicalScalingMultisampleDepth);
+        OpenGLFuncs.glBindRenderbuffer(GL_RENDERBUFFER, 0);
+
+        if ( (OpenGLFuncs.glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) || OpenGLFuncs.glGetError() ) {
+            OpenGLFuncs.glBindFramebuffer(GL_FRAMEBUFFER, 0);
+            OpenGLFuncs.glDeleteRenderbuffers(1, &OpenGLLogicalScalingMultisampleColor);
+            OpenGLFuncs.glDeleteRenderbuffers(1, &OpenGLLogicalScalingMultisampleDepth);
+            OpenGLFuncs.glDeleteFramebuffers(1, &OpenGLLogicalScalingMultisampleFBO);
+            OpenGLLogicalScalingMultisampleFBO = OpenGLLogicalScalingMultisampleColor = OpenGLLogicalScalingMultisampleDepth = 0;
+        }
+    }
+
+    OpenGLFuncs.glBindFramebuffer(GL_FRAMEBUFFER, OpenGLLogicalScalingFBO);
+
     OpenGLFuncs.glViewport(0, 0, w, h);
     OpenGLFuncs.glScissor(0, 0, w, h);
     OpenGLLogicalScalingWidth = w;
     OpenGLLogicalScalingHeight = h;
+
     OpenGLFuncs.glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
     return SDL_TRUE;
 }
@@ -3462,6 +3491,9 @@ SDL_SetVideoMode(int width, int height, int bpp, Uint32 flags12)
         OpenGLLogicalScalingFBO = 0;
         OpenGLLogicalScalingColor = 0;
         OpenGLLogicalScalingDepth = 0;
+        OpenGLLogicalScalingMultisampleFBO = 0;
+        OpenGLLogicalScalingMultisampleColor = 0;
+        OpenGLLogicalScalingMultisampleDepth = 0;
     }
 
     if (flags12 & SDL12_FULLSCREEN) {
@@ -3488,6 +3520,12 @@ SDL_SetVideoMode(int width, int height, int bpp, Uint32 flags12)
            ones did (x11, windib, quartz), so we'll just offer it everywhere. */
         GetEnvironmentWindowPosition(&x, &y);
 
+        /* If GL scaling is disabled, and a multisampled buffer is requested, do it. */
+        if (!use_gl_scaling && (flags12 & SDL12_OPENGL) && OpenGLLogicalScalingSamples) {
+            SDL20_GL_SetAttribute(SDL_GL_MULTISAMPLEBUFFERS, 1);
+            SDL20_GL_SetAttribute(SDL_GL_MULTISAMPLESAMPLES, OpenGLLogicalScalingSamples);
+        }
+
         VideoWindow20 = SDL20_CreateWindow(WindowTitle, x, y, width, height, flags20);
         if (!VideoWindow20) {
             return EndVidModeCreate();
@@ -4633,6 +4671,13 @@ SDL_GL_SetAttribute(SDL12_GLattr attr, int value)
         FIXME("Actually set swap interval somewhere");
         return 0;
     }
+    else if (attr == SDL12_GL_MULTISAMPLESAMPLES) {
+        OpenGLLogicalScalingSamples = value;
+        return 0;
+    }
+    else if (attr == SDL12_GL_MULTISAMPLEBUFFERS) {
+        return 0;
+    }
     return SDL20_GL_SetAttribute((SDL_GLattr) attr, value);
 }
 
@@ -4647,6 +4692,14 @@ SDL_GL_GetAttribute(SDL12_GLattr attr, int* value)
         *value = SDL20_GL_GetSwapInterval();
         return 0;
     }
+    else if (attr == SDL12_GL_MULTISAMPLESAMPLES) {
+        *value = OpenGLLogicalScalingSamples;
+        return 0;
+    }        
+    else if (attr == SDL12_GL_MULTISAMPLEBUFFERS) {
+        *value = (OpenGLLogicalScalingSamples) ? 1 : 0;
+        return 0;
+    }        
     return SDL20_GL_GetAttribute((SDL_GLattr) attr, value);
 }
 
@@ -4664,6 +4717,10 @@ SDL_GL_SwapBuffers(void)
             int drawablew, drawableh;
             SDL_Rect dstrect;
 
+            GLint old_draw_fbo, old_read_fbo;
+            OpenGLFuncs.glGetIntegerv(GL_DRAW_FRAMEBUFFER_BINDING, &old_draw_fbo);
+            OpenGLFuncs.glGetIntegerv(GL_READ_FRAMEBUFFER_BINDING, &old_read_fbo);
+
             SDL20_GL_GetDrawableSize(VideoWindow20, &drawablew, &drawableh);
             OpenGLFuncs.glGetFloatv(GL_COLOR_CLEAR_VALUE, clearcolor);
 
@@ -4692,11 +4749,22 @@ SDL_GL_SwapBuffers(void)
                 dstrect.x = (drawablew - dstrect.w) / 2;
             }
 
-            OpenGLFuncs.glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
-            OpenGLFuncs.glBindFramebuffer(GL_READ_FRAMEBUFFER, OpenGLLogicalScalingFBO);
             if (has_scissor) {
                 OpenGLFuncs.glDisable(GL_SCISSOR_TEST);  /* scissor test affects framebuffer_blit */
             }
+
+            OpenGLFuncs.glBindFramebuffer(GL_READ_FRAMEBUFFER, OpenGLLogicalScalingFBO);
+
+            /* Resolve the multisample framebuffer if required. */
+            if (OpenGLLogicalScalingMultisampleFBO) {
+                OpenGLFuncs.glBindFramebuffer(GL_DRAW_FRAMEBUFFER, OpenGLLogicalScalingMultisampleFBO);
+                OpenGLFuncs.glBlitFramebuffer(0, 0, OpenGLLogicalScalingWidth, OpenGLLogicalScalingHeight,
+                                              0, 0, OpenGLLogicalScalingWidth, OpenGLLogicalScalingHeight,
+                                              GL_COLOR_BUFFER_BIT, GL_NEAREST);
+                OpenGLFuncs.glBindFramebuffer(GL_READ_FRAMEBUFFER, OpenGLLogicalScalingMultisampleFBO);
+            }
+
+            OpenGLFuncs.glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
             OpenGLFuncs.glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
             OpenGLFuncs.glClear(GL_COLOR_BUFFER_BIT);
             OpenGLFuncs.glBlitFramebuffer(0, 0, OpenGLLogicalScalingWidth, OpenGLLogicalScalingHeight,
@@ -4708,7 +4776,8 @@ SDL_GL_SwapBuffers(void)
             if (has_scissor) {
                 OpenGLFuncs.glEnable(GL_SCISSOR_TEST);
             }
-            OpenGLFuncs.glBindFramebuffer(GL_FRAMEBUFFER, OpenGLLogicalScalingFBO);
+            OpenGLFuncs.glBindFramebuffer(GL_READ_FRAMEBUFFER, old_read_fbo);
+            OpenGLFuncs.glBindFramebuffer(GL_DRAW_FRAMEBUFFER, old_draw_fbo);
         } else {
             SDL20_GL_SwapWindow(VideoWindow20);
         }
diff --git a/src/SDL20_syms.h b/src/SDL20_syms.h
index 861b07d..a2a512a 100644
--- a/src/SDL20_syms.h
+++ b/src/SDL20_syms.h
@@ -317,6 +317,7 @@ OPENGL_SYM(Core,GLboolean,glIsEnabled,(GLenum a),(a),return)
 OPENGL_SYM(Core,void,glEnable,(GLenum a),(a),)
 OPENGL_SYM(Core,void,glDisable,(GLenum a),(a),)
 OPENGL_SYM(Core,void,glGetFloatv,(GLenum a, GLfloat *b),(a,b),)
+OPENGL_SYM(Core,void,glGetIntegerv,(GLenum a, GLint *b),(a,b),)
 OPENGL_SYM(Core,void,glClearColor,(GLfloat a,GLfloat b,GLfloat c,GLfloat d),(a,b,c,d),)
 OPENGL_SYM(Core,void,glViewport,(GLint a, GLint b, GLsizei c, GLsizei d),(a,b,c,d),)
 OPENGL_SYM(Core,void,glScissor,(GLint a, GLint b, GLsizei c, GLsizei d),(a,b,c,d),)
@@ -326,6 +327,7 @@ OPENGL_SYM(GL_ARB_framebuffer_object,void,glBindRenderbuffer,(GLenum a, GLuint b
 OPENGL_SYM(GL_ARB_framebuffer_object,void,glDeleteRenderbuffers,(GLsizei a, const GLuint *b),(a,b),)
 OPENGL_SYM(GL_ARB_framebuffer_object,void,glGenRenderbuffers,(GLsizei a, GLuint *b),(a,b),)
 OPENGL_SYM(GL_ARB_framebuffer_object,void,glRenderbufferStorage,(GLenum a, GLenum b, GLsizei c, GLsizei d),(a,b,c,d),)
+OPENGL_SYM(GL_ARB_framebuffer_object,void,glRenderbufferStorageMultisample,(GLenum a, GLsizei b, GLenum c, GLsizei d, GLsizei e),(a,b,c,d,e),)
 OPENGL_SYM(GL_ARB_framebuffer_object,void,glGetRenderbufferParameteriv,(GLenum a, GLenum b, GLint* c),(a,b,c),)
 OPENGL_SYM(GL_ARB_framebuffer_object,GLboolean,glIsFramebuffer,(GLuint a),(a),return)
 OPENGL_SYM(GL_ARB_framebuffer_object,void,glBindFramebuffer,(GLenum a, GLuint b),(a,b),)