SDL_shader_tools: Emit error when encountering an unknown preprocessor directive

From 0d14ad4f5d708678d3c284853ad8a641743b89a3 Mon Sep 17 00:00:00 2001
From: Anonymous Maarten <[EMAIL REDACTED]>
Date: Thu, 21 Sep 2023 07:34:37 +0200
Subject: [PATCH] Emit error when encountering an unknown preprocessor
 directive

---
 .github/workflows/main.yml                    |  22 +-
 CMakeLists.txt                                |  11 +-
 SDL_shader_internal.h                         |   2 +
 SDL_shader_lexer.c                            | 520 ++++++++++++------
 SDL_shader_lexer.re                           |   9 +-
 SDL_shader_preprocessor.c                     |  20 +-
 .../preprocessor/errors/unknown-directive     |   1 +
 .../errors/unknown-directive.correct          |   1 +
 .../preprocessor/output/multiline-concat      |   3 +
 .../output/multiline-concat.correct           |   1 +
 .../preprocessor/output/multiline-spaces      |   3 +
 .../output/multiline-spaces.correct           |   1 +
 12 files changed, 411 insertions(+), 183 deletions(-)
 create mode 100644 unit_tests/preprocessor/errors/unknown-directive
 create mode 100644 unit_tests/preprocessor/errors/unknown-directive.correct
 create mode 100644 unit_tests/preprocessor/output/multiline-concat
 create mode 100644 unit_tests/preprocessor/output/multiline-concat.correct
 create mode 100644 unit_tests/preprocessor/output/multiline-spaces
 create mode 100644 unit_tests/preprocessor/output/multiline-spaces.correct

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 7f98f82..b9cdf82 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -8,20 +8,30 @@ jobs:
     runs-on: ${{ matrix.platform.os }}
     strategy:
       matrix:
-        platform:  # !!! FIXME: figure out an efficient way to get SDL2 on the Windows/Mac bots.
-        - { name: Linux,   os: ubuntu-latest, flags: -GNinja }
+        platform:
+        - { name: Linux,   os: ubuntu-latest }
         #- { name: Windows, os: windows-latest }
         #- { name: MacOS,   os: macos-latest }
     steps:
+    - name: Install Ninja
+      uses: turtlesec-no/get-ninja@main
+    - name: Set up SDL
+      id: sdl
+      uses: libsdl-org/setup-sdl@main
+      with:
+        cmake-generator: Ninja
+        version: 2-head
     - name: Setup Linux dependencies
-      if: runner.os == 'Linux'
+      if: ${{ runner.os == 'Linux' }}
       run: |
+        # !!! FIXME: figure out an efficient way to get SDL2 on the Windows/Mac bots.
         sudo apt-get update
-        sudo apt-get install cmake ninja-build libsdl2-dev re2c
+        sudo apt-get install re2c
     - name: Get SDL_shader_tools sources
       uses: actions/checkout@v2
     - name: Configure CMake
-      run: cmake -B build ${{ matrix.platform.flags }}
+      run: cmake -B build -GNinja -DSDLSL_TESTS=ON
     - name: Build
       run: cmake --build build/
-
+    - name: Test
+      run: ctest -V --test-dir build --no-tests=error
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 60541c0..cb5b73d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,8 +1,10 @@
-cmake_minimum_required(VERSION 3.16)
+ cmake_minimum_required(VERSION 3.16)
 project(SDL_shader_tools LANGUAGES C)
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake")
 
+option(SDLSL_TESTS "Build SDL_shader_tools tests" OFF)
+
 include(ExternalProject)
 
 if(CMAKE_CROSSCOMPILING)
@@ -81,3 +83,10 @@ target_include_directories(sdl-shader-bytecode-dumper PRIVATE "${CMAKE_CURRENT_S
 target_link_libraries(sdl-shader-bytecode-dumper PRIVATE SDL2::SDL2)
 target_include_directories(sdl-shader-bytecode-dumper PRIVATE ${SDL2_INCLUDE_DIRS} ${SDL2_INCLUDE_DIR})
 target_compile_definitions(sdl-shader-bytecode-dumper PRIVATE SDL_MAIN_HANDLED)
+
+find_package(Perl)
+if(SDLSL_TESTS AND NOT CMAKE_CROSSCOMPILING AND Perl_FOUND)
+    enable_testing()
+    add_test(NAME SDL_shader_tools-tests COMMAND ${PERL_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/unit_tests/run_tests.pl")
+    set_tests_properties(SDL_shader_tools-tests PROPERTIES WORKING_DIRECTORY "$<TARGET_FILE_DIR:sdl-shader-compiler>")
+endif()
diff --git a/SDL_shader_internal.h b/SDL_shader_internal.h
index 16a6edd..bc608a5 100644
--- a/SDL_shader_internal.h
+++ b/SDL_shader_internal.h
@@ -207,6 +207,7 @@ typedef enum
     TOKEN_PP_ENDIF,
     TOKEN_PP_ERROR,  /* caught in the preprocessor, not passed to caller */
     TOKEN_PP_PRAGMA,
+    TOKEN_PP_BAD,
     TOKEN_INCOMPLETE_STRING_LITERAL,
     TOKEN_INCOMPLETE_COMMENT,
     TOKEN_PP_UNARY_MINUS,  /* used internally, never returned. */
@@ -249,6 +250,7 @@ typedef struct IncludeState
     const unsigned char *lexer_marker;
     SDL_bool report_whitespace;
     SDL_bool asm_comments;
+    SDL_bool expanding_macro;
     size_t orig_length;
     size_t bytes_left;
     Sint32 line;
diff --git a/SDL_shader_lexer.c b/SDL_shader_lexer.c
index 3b81839..214686d 100644
--- a/SDL_shader_lexer.c
+++ b/SDL_shader_lexer.c
@@ -23,7 +23,7 @@
 
 typedef unsigned char uchar;
 
-#define YYMAXFILL 8
+#define YYMAXFILL 9
 
 #define RET(t) return update_state(s, eoi, cursor, token, (Token) t)
 #define YYCTYPE uchar
@@ -64,8 +64,8 @@ Token preprocessor_lexer(IncludeState *s)
 
 
 
-    // preprocessor directives are only valid at start of line.
-    if (s->tokenval == ((Token) '\n')) {
+    // preprocessor directives are only valid at start of line, and when not expanding a macro.
+    if (s->tokenval == ((Token) '\n') && !s->expanding_macro) {
         goto ppdirective;  // may jump back to scanner_loop.
     }
 
@@ -762,8 +762,7 @@ Token preprocessor_lexer(IncludeState *s)
 
 {
 	YYCTYPE yych;
-	unsigned int yyaccept = 0;
-	if ((YYLIMIT - YYCURSOR) < 8) YYFILL(8);
+	if ((YYLIMIT - YYCURSOR) < 9) YYFILL(9);
 	yych = *YYCURSOR;
 	if (yych <= '\f') {
 		if (yych <= 0x08) {
@@ -783,11 +782,7 @@ Token preprocessor_lexer(IncludeState *s)
 	}
 yy106:
 	YYCURSOR = YYMARKER;
-	if (yyaccept == 0) {
-		goto yy108;
-	} else {
-		goto yy124;
-	}
+	goto yy108;
 yy107:
 	++YYCURSOR;
 yy108:
@@ -808,255 +803,442 @@ Token preprocessor_lexer(IncludeState *s)
 	}
 	{ goto ppdirective; }
 yy110:
-	yyaccept = 0;
 	yych = *(YYMARKER = ++YYCURSOR);
-	if (yych <= 'h') {
-		if (yych <= 0x1F) {
-			if (yych == '\t') goto yy112;
-			goto yy108;
+	if (yych <= '@') {
+		if (yych <= '\t') {
+			if (yych <= 0x08) goto yy108;
+			goto yy112;
 		} else {
-			if (yych <= ' ') goto yy112;
-			if (yych <= 'c') goto yy108;
-			if (yych <= 'e') goto yy112;
+			if (yych == ' ') goto yy112;
 			goto yy108;
 		}
 	} else {
-		if (yych <= 'o') {
-			if (yych <= 'i') goto yy112;
-			if (yych == 'l') goto yy112;
-			goto yy108;
+		if (yych <= '_') {
+			if (yych <= 'Z') goto yy112;
+			if (yych <= '^') goto yy108;
+			goto yy112;
 		} else {
-			if (yych <= 'p') goto yy112;
-			if (yych == 'u') goto yy112;
+			if (yych <= '`') goto yy108;
+			if (yych <= 'z') goto yy112;
 			goto yy108;
 		}
 	}
 yy111:
 	++YYCURSOR;
-	if ((YYLIMIT - YYCURSOR) < 7) YYFILL(7);
+	if ((YYLIMIT - YYCURSOR) < 8) YYFILL(8);
 	yych = *YYCURSOR;
 yy112:
-	if (yych <= 'h') {
-		if (yych <= ' ') {
-			if (yych == '\t') goto yy111;
-			if (yych <= 0x1F) goto yy106;
-			goto yy111;
+	if (yych <= 'd') {
+		if (yych <= '@') {
+			if (yych <= '\t') {
+				if (yych <= 0x08) goto yy106;
+				goto yy111;
+			} else {
+				if (yych == ' ') goto yy111;
+				goto yy106;
+			}
 		} else {
-			if (yych <= 'c') goto yy106;
-			if (yych <= 'd') goto yy113;
-			if (yych <= 'e') goto yy114;
-			goto yy106;
+			if (yych <= '_') {
+				if (yych <= 'Z') goto yy113;
+				if (yych <= '^') goto yy106;
+			} else {
+				if (yych <= '`') goto yy106;
+				if (yych >= 'd') goto yy116;
+			}
 		}
 	} else {
-		if (yych <= 'o') {
-			if (yych <= 'i') goto yy115;
-			if (yych == 'l') goto yy116;
-			goto yy106;
+		if (yych <= 'l') {
+			if (yych <= 'h') {
+				if (yych <= 'e') goto yy117;
+			} else {
+				if (yych <= 'i') goto yy118;
+				if (yych >= 'l') goto yy119;
+			}
 		} else {
-			if (yych <= 'p') goto yy117;
-			if (yych == 'u') goto yy118;
-			goto yy106;
+			if (yych <= 't') {
+				if (yych == 'p') goto yy120;
+			} else {
+				if (yych <= 'u') goto yy121;
+				if (yych >= '{') goto yy106;
+			}
 		}
 	}
 yy113:
-	yych = *++YYCURSOR;
-	if (yych == 'e') goto yy119;
-	goto yy106;
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
 yy114:
-	yych = *++YYCURSOR;
-	if (yych <= 'm') {
-		if (yych == 'l') goto yy120;
-		goto yy106;
+	if (yych <= 'Z') {
+		if (yych <= '/') goto yy115;
+		if (yych <= '9') goto yy113;
+		if (yych >= 'A') goto yy113;
 	} else {
-		if (yych <= 'n') goto yy121;
-		if (yych == 'r') goto yy122;
-		goto yy106;
+		if (yych <= '_') {
+			if (yych >= '_') goto yy113;
+		} else {
+			if (yych <= '`') goto yy115;
+			if (yych <= 'z') goto yy113;
+		}
 	}
 yy115:
-	yych = *++YYCURSOR;
-	if (yych == 'f') goto yy123;
-	if (yych == 'n') goto yy125;
-	goto yy106;
+	{ RET(TOKEN_PP_BAD); }
 yy116:
 	yych = *++YYCURSOR;
-	if (yych == 'i') goto yy126;
-	goto yy106;
+	if (yych == 'e') goto yy122;
+	goto yy114;
 yy117:
 	yych = *++YYCURSOR;
-	if (yych == 'r') goto yy127;
-	goto yy106;
+	if (yych <= 'm') {
+		if (yych == 'l') goto yy123;
+		goto yy114;
+	} else {
+		if (yych <= 'n') goto yy124;
+		if (yych == 'r') goto yy125;
+		goto yy114;
+	}
 yy118:
 	yych = *++YYCURSOR;
+	if (yych == 'f') goto yy126;
 	if (yych == 'n') goto yy128;
-	goto yy106;
+	goto yy114;
 yy119:
 	yych = *++YYCURSOR;
-	if (yych == 'f') goto yy129;
-	goto yy106;
+	if (yych == 'i') goto yy129;
+	goto yy114;
 yy120:
 	yych = *++YYCURSOR;
-	if (yych == 'i') goto yy130;
-	if (yych == 's') goto yy131;
-	goto yy106;
+	if (yych == 'r') goto yy130;
+	goto yy114;
 yy121:
 	yych = *++YYCURSOR;
-	if (yych == 'd') goto yy132;
-	goto yy106;
+	if (yych == 'n') goto yy131;
+	goto yy114;
 yy122:
 	yych = *++YYCURSOR;
-	if (yych == 'r') goto yy133;
-	goto yy106;
+	if (yych == 'f') goto yy132;
+	goto yy114;
 yy123:
-	yyaccept = 1;
-	yych = *(YYMARKER = ++YYCURSOR);
-	if (yych == 'd') goto yy134;
-	if (yych == 'n') goto yy135;
+	yych = *++YYCURSOR;
+	if (yych == 'i') goto yy133;
+	if (yych == 's') goto yy134;
+	goto yy114;
 yy124:
-	{ RET(TOKEN_PP_IF); }
+	yych = *++YYCURSOR;
+	if (yych == 'd') goto yy135;
+	goto yy114;
 yy125:
 	yych = *++YYCURSOR;
-	if (yych == 'c') goto yy136;
-	goto yy106;
+	if (yych == 'r') goto yy136;
+	goto yy114;
 yy126:
 	yych = *++YYCURSOR;
-	if (yych == 'n') goto yy137;
-	goto yy106;
+	if (yych <= '_') {
+		if (yych <= '@') {
+			if (yych <= '/') goto yy127;
+			if (yych <= '9') goto yy113;
+		} else {
+			if (yych <= 'Z') goto yy113;
+			if (yych >= '_') goto yy113;
+		}
+	} else {
+		if (yych <= 'd') {
+			if (yych <= '`') goto yy127;
+			if (yych <= 'c') goto yy113;
+			goto yy137;
+		} else {
+			if (yych == 'n') goto yy138;
+			if (yych <= 'z') goto yy113;
+		}
+	}
 yy127:
-	yych = *++YYCURSOR;
-	if (yych == 'a') goto yy138;
-	goto yy106;
+	{ RET(TOKEN_PP_IF); }
 yy128:
 	yych = *++YYCURSOR;
-	if (yych == 'd') goto yy139;
-	goto yy106;
+	if (yych == 'c') goto yy139;
+	goto yy114;
 yy129:
 	yych = *++YYCURSOR;
-	if (yych == 'i') goto yy140;
-	goto yy106;
+	if (yych == 'n') goto yy140;
+	goto yy114;
 yy130:
 	yych = *++YYCURSOR;
-	if (yych == 'f') goto yy141;
-	goto yy106;
+	if (yych == 'a') goto yy141;
+	goto yy114;
 yy131:
 	yych = *++YYCURSOR;
-	if (yych == 'e') goto yy142;
-	goto yy106;
+	if (yych == 'd') goto yy142;
+	goto yy114;
 yy132:
 	yych = *++YYCURSOR;
 	if (yych == 'i') goto yy143;
-	goto yy106;
+	goto yy114;
 yy133:
 	yych = *++YYCURSOR;
-	if (yych == 'o') goto yy144;
-	goto yy106;
+	if (yych == 'f') goto yy144;
+	goto yy114;
 yy134:
 	yych = *++YYCURSOR;
-	if (yych == 'e') goto yy145;
-	goto yy106;
+	if (yych == 'e') goto yy146;
+	goto yy114;
 yy135:
 	yych = *++YYCURSOR;
-	if (yych == 'd') goto yy146;
-	goto yy106;
+	if (yych == 'i') goto yy148;
+	goto yy114;
 yy136:
 	yych = *++YYCURSOR;
-	if (yych == 'l') goto yy147;
-	goto yy106;
+	if (yych == 'o') goto yy149;
+	goto yy114;
 yy137:
 	yych = *++YYCURSOR;
-	if (yych == 'e') goto yy148;
-	goto yy106;
+	if (yych == 'e') goto yy150;
+	goto yy114;
 yy138:
 	yych = *++YYCURSOR;
-	if (yych == 'g') goto yy149;
-	goto yy106;
+	if (yych == 'd') goto yy151;
+	goto yy114;
 yy139:
 	yych = *++YYCURSOR;
-	if (yych == 'e') goto yy150;
-	goto yy106;
+	if (yych == 'l') goto yy152;
+	goto yy114;
 yy140:
 	yych = *++YYCURSOR;
-	if (yych == 'n') goto yy151;
-	goto yy106;
+	if (yych == 'e') goto yy153;
+	goto yy114;
 yy141:
-	++YYCURSOR;
-	{ RET(TOKEN_PP_ELIF); }
+	yych = *++YYCURSOR;
+	if (yych == 'g') goto yy155;
+	goto yy114;
 yy142:
-	++YYCURSOR;
-	{ RET(TOKEN_PP_ELSE); }
+	yych = *++YYCURSOR;
+	if (yych == 'e') goto yy156;
+	goto yy114;
 yy143:
 	yych = *++YYCURSOR;
-	if (yych == 'f') goto yy152;
-	goto yy106;
+	if (yych == 'n') goto yy157;
+	goto yy114;
 yy144:
 	yych = *++YYCURSOR;
-	if (yych == 'r') goto yy153;
-	goto yy106;
+	if (yych <= 'Z') {
+		if (yych <= '/') goto yy145;
+		if (yych <= '9') goto yy113;
+		if (yych >= 'A') goto yy113;
+	} else {
+		if (yych <= '_') {
+			if (yych >= '_') goto yy113;
+		} else {
+			if (yych <= '`') goto yy145;
+			if (yych <= 'z') goto yy113;
+		}
+	}
 yy145:
-	yych = *++YYCURSOR;
-	if (yych == 'f') goto yy154;
-	goto yy106;
+	{ RET(TOKEN_PP_ELIF); }
 yy146:
 	yych = *++YYCURSOR;
-	if (yych == 'e') goto yy155;
-	goto yy106;
+	if (yych <= 'Z') {
+		if (yych <= '/') goto yy147;
+		if (yych <= '9') goto yy113;
+		if (yych >= 'A') goto yy113;
+	} else {
+		if (yych <= '_') {
+			if (yych >= '_') goto yy113;
+		} else {
+			if (yych <= '`') goto yy147;
+			if (yych <= 'z') goto yy113;
+		}
+	}
 yy147:
-	yych = *++YYCURSOR;
-	if (yych == 'u') goto yy156;
-	goto yy106;
+	{ RET(TOKEN_PP_ELSE); }
 yy148:
-	++YYCURSOR;
-	{ RET(TOKEN_PP_LINE); }
+	yych = *++YYCURSOR;
+	if (yych == 'f') goto yy158;
+	goto yy114;
 yy149:
 	yych = *++YYCURSOR;
-	if (yych == 'm') goto yy157;
-	goto yy106;
+	if (yych == 'r') goto yy160;
+	goto yy114;
 yy150:
 	yych = *++YYCURSOR;
-	if (yych == 'f') goto yy158;
-	goto yy106;
+	if (yych == 'f') goto yy162;
+	goto yy114;
 yy151:
 	yych = *++YYCURSOR;
-	if (yych == 'e') goto yy159;
-	goto yy106;
+	if (yych == 'e') goto yy164;
+	goto yy114;
 yy152:
-	++YYCURSOR;
-	{ RET(TOKEN_PP_ENDIF); }
+	yych = *++YYCURSOR;
+	if (yych == 'u') goto yy165;
+	goto yy114;
 yy153:
-	++YYCURSOR;
-	{ RET(TOKEN_PP_ERROR); }
+	yych = *++YYCURSOR;
+	if (yych <= 'Z') {
+		if (yych <= '/') goto yy154;
+		if (yych <= '9') goto yy113;
+		if (yych >= 'A') goto yy113;
+	} else {
+		if (yych <= '_') {
+			if (yych >= '_') goto yy113;
+		} else {
+			if (yych <= '`') goto yy154;
+			if (yych <= 'z') goto yy113;
+		}
+	}
 yy154:
-	++YYCURSOR;
-	{ RET(TOKEN_PP_IFDEF); }
+	{ RET(TOKEN_PP_LINE); }
 yy155:
 	yych = *++YYCURSOR;
-	if (yych == 'f') goto yy160;
-	goto yy106;
+	if (yych == 'm') goto yy166;
+	goto yy114;
 yy156:
 	yych = *++YYCURSOR;
-	if (yych == 'd') goto yy161;
-	goto yy106;
+	if (yych == 'f') goto yy167;
+	goto yy114;
 yy157:
 	yych = *++YYCURSOR;
-	if (yych == 'a') goto yy162;
-	goto yy106;
+	if (yych == 'e') goto yy169;
+	goto yy114;
 yy158:
-	++YYCURSOR;
-	{ RET(TOKEN_PP_UNDEF); }
+	yych = *++YYCURSOR;
+	if (yych <= 'Z') {
+		if (yych <= '/') goto yy159;
+		if (yych <= '9') goto yy113;
+		if (yych >= 'A') goto yy113;
+	} else {
+		if (yych <= '_') {
+			if (yych >= '_') goto yy113;
+		} else {
+			if (yych <= '`') goto yy159;
+			if (yych <= 'z') goto yy113;
+		}
+	}
 yy159:
-	++YYCURSOR;
-	{ RET(TOKEN_PP_DEFINE); }
+	{ RET(TOKEN_PP_ENDIF); }
 yy160:
-	++YYCURSOR;
-	{ RET(TOKEN_PP_IFNDEF); }
-yy161:
 	yych = *++YYCURSOR;
-	if (yych == 'e') goto yy163;
-	goto yy106;
+	if (yych <= 'Z') {
+		if (yych <= '/') goto yy161;
+		if (yych <= '9') goto yy113;
+		if (yych >= 'A') goto yy113;
+	} else {
+		if (yych <= '_') {
+			if (yych >= '_') goto yy113;
+		} else {
+			if (yych <= '`') goto yy161;
+			if (yych <= 'z') goto yy113;
+		}
+	}
+yy161:
+	{ RET(TOKEN_PP_ERROR); }
 yy162:
-	++YYCURSOR;
-	{ RET(TOKEN_PP_PRAGMA); }
+	yych = *++YYCURSOR;
+	if (yych <= 'Z') {
+		if (yych <= '/') goto yy163;
+		if (yych <= '9') goto yy113;
+		if (yych >= 'A') goto yy113;
+	} else {
+		if (yych <= '_') {
+			if (yych >= '_') goto yy113;
+		} else {
+			if (yych <= '`') goto yy163;
+			if (yych <= 'z') goto yy113;
+		}
+	}
 yy163:
-	++YYCURSOR;
+	{ RET(TOKEN_PP_IFDEF); }
+yy164:
+	yych = *++YYCURSOR;
+	if (yych == 'f') goto yy171;
+	goto yy114;
+yy165:
+	yych = *++YYCURSOR;
+	if (yych == 'd') goto yy173;
+	goto yy114;
+yy166:
+	yych = *++YYCURSOR;
+	if (yych == 'a') goto yy174;
+	goto yy114;
+yy167:
+	yych = *++YYCURSOR;
+	if (yych <= 'Z') {
+		if (yych <= '/') goto yy168;
+		if (yych <= '9') goto yy113;
+		if (yych >= 'A') goto yy113;
+	} else {
+		if (yych <= '_') {
+			if (yych >= '_') goto yy113;
+		} else {
+			if (yych <= '`') goto yy168;
+			if (yych <= 'z') goto yy113;
+		}
+	}
+yy168:
+	{ RET(TOKEN_PP_UNDEF); }
+yy169:
+	yych = *++YYCURSOR;
+	if (yych <= 'Z') {
+		if (yych <= '/') goto yy170;
+		if (yych <= '9') goto yy113;
+		if (yych >= 'A') goto yy113;
+	} else {
+		if (yych <= '_') {
+			if (yych >= '_') goto yy113;
+		} else {
+			if (yych <= '`') goto yy170;
+			if (yych <= 'z') goto yy113;
+		}
+	}
+yy170:
+	{ RET(TOKEN_PP_DEFINE); }
+yy171:
+	yych = *++YYCURSOR;
+	if (yych <= 'Z') {
+		if (yych <= '/') goto yy172;
+		if (yych <= '9') goto yy113;
+		if (yych >= 'A') goto yy113;
+	} else {
+		if (yych <= '_') {
+			if (yych >= '_') goto yy113;
+		} else {
+			if (yych <= '`') goto yy172;
+			if (yych <= 'z') goto yy113;
+		}
+	}
+yy172:
+	{ RET(TOKEN_PP_IFNDEF); }
+yy173:
+	yych = *++YYCURSOR;
+	if (yych == 'e') goto yy176;
+	goto yy114;
+yy174:
+	yych = *++YYCURSOR;
+	if (yych <= 'Z') {
+		if (yych <= '/') goto yy175;
+		if (yych <= '9') goto yy113;
+		if (yych >= 'A') goto yy113;
+	} else {
+		if (yych <= '_') {
+			if (yych >= '_') goto yy113;
+		} else {
+			if (yych <= '`') goto yy175;
+			if (yych <= 'z') goto yy113;
+		}
+	}
+yy175:
+	{ RET(TOKEN_PP_PRAGMA); }
+yy176:
+	yych = *++YYCURSOR;
+	if (yych <= 'Z') {
+		if (yych <= '/') goto yy177;
+		if (yych <= '9') goto yy113;
+		if (yych >= 'A') goto yy113;
+	} else {
+		if (yych <= '_') {
+			if (yych >= '_') goto yy113;
+		} else {
+			if (yych <= '`') goto yy177;
+			if (yych <= 'z') goto yy113;
+		}
+	}
+yy177:
 	{ RET(TOKEN_PP_INCLUDE); }
 }
 
@@ -1070,26 +1252,26 @@ Token preprocessor_lexer(IncludeState *s)
 	yych = *YYCURSOR;
 	if (yych <= '#') {
 		if (yych <= '\r') {
-			if (yych <= 0x00) goto yy165;
-			if (yych <= 0x08) goto yy166;
-			goto yy167;
+			if (yych <= 0x00) goto yy179;
+			if (yych <= 0x08) goto yy180;
+			goto yy181;
 		} else {
-			if (yych <= 0x1F) goto yy166;
-			if (yych == '"') goto yy166;
-			goto yy167;
+			if (yych <= 0x1F) goto yy180;
+			if (yych == '"') goto yy180;
+			goto yy181;
 		}
 	} else {
 		if (yych <= '@') {
-			if (yych <= '$') goto yy166;
-			if (yych <= '?') goto yy167;
-			goto yy166;
+			if (yych <= '$') goto yy180;
+			if (yych <= '?') goto yy181;
+			goto yy180;
 		} else {
-			if (yych == '`') goto yy166;
-			if (yych <= '~') goto yy167;
-			goto yy166;
+			if (yych == '`') goto yy180;
+			if (yych <= '~') goto yy181;
+			goto yy180;
 		}
 	}
-yy165:
+yy179:
 	++YYCURSOR;
 	{
                         if (eoi)
@@ -1101,10 +1283,10 @@ Token preprocessor_lexer(IncludeState *s)
                         }
                         goto bad_chars;
                     }
-yy166:
+yy180:
 	++YYCURSOR;
 	{ goto bad_chars; }
-yy167:
+yy181:
 	++YYCURSOR;
 	{ cursor--; RET(TOKEN_BAD_CHARS); }
 }
diff --git a/SDL_shader_lexer.re b/SDL_shader_lexer.re
index c8f64fb..bcb9235 100644
--- a/SDL_shader_lexer.re
+++ b/SDL_shader_lexer.re
@@ -75,8 +75,8 @@ Token preprocessor_lexer(IncludeState *s)
     QUOTE = ["];
 */
 
-    // preprocessor directives are only valid at start of line.
-    if (s->tokenval == ((Token) '\n')) {
+    // preprocessor directives are only valid at start of line, and when not expanding a macro.
+    if (s->tokenval == ((Token) '\n') && !s->expanding_macro) {
         goto ppdirective;  // may jump back to scanner_loop.
     }
 
@@ -99,11 +99,11 @@ scanner_loop:
     QUOTE           { goto stringliteral; }
 
     L (L|D)*        { RET(TOKEN_IDENTIFIER); }
-    
+
     ("0" [xX] H+) | ("0" D+) | (D+) |
     (['] (ESC|ANY\[\r\n\\'])* ['])
                     { RET(TOKEN_INT_LITERAL); }
-    
+
     (D+ E) | (D* "." D+ E?) | (D+ "." D* E?)
                     { RET(TOKEN_FLOAT_LITERAL); }
 
@@ -229,6 +229,7 @@ ppdirective:
         PP "endif"      { RET(TOKEN_PP_ENDIF); }
         PP "error"      { RET(TOKEN_PP_ERROR); }
         PP "pragma"     { RET(TOKEN_PP_PRAGMA); }
+        PP L (L|D)*     { RET(TOKEN_PP_BAD); }
         WHITESPACE      { goto ppdirective; }
 
         ANY             {
diff --git a/SDL_shader_preprocessor.c b/SDL_shader_preprocessor.c
index 4d16d0b..558d29c 100644
--- a/SDL_shader_preprocessor.c
+++ b/SDL_shader_preprocessor.c
@@ -90,6 +90,7 @@ void SDL_SHADER_print_debug_token(const char *subsystem, const char *token,
         TOKENCASE(TOKEN_PP_ENDIF);
         TOKENCASE(TOKEN_PP_ERROR);
         TOKENCASE(TOKEN_PP_PRAGMA);
+        TOKENCASE(TOKEN_PP_BAD);
         TOKENCASE(TOKEN_INCOMPLETE_STRING_LITERAL);
         TOKENCASE(TOKEN_INCOMPLETE_COMMENT);
         TOKENCASE(TOKEN_BAD_CHARS);
@@ -128,7 +129,7 @@ static const char *attempt_include_open(const char *path, const char *fname,
     char *fullpath = (char *) m(len, d);
     if (fullpath == NULL) {
         SDL_snprintf(failstr, failstrlen, "Out of memory");
-        return SDL_FALSE;
+        return NULL;
     }
 
     *failstr = '\0';
@@ -205,7 +206,7 @@ static const char *internal_include_open(SDL_SHADER_IncludeType inctype,
         char *parent_dir = m(slen, d);
         if (!parent_dir) {
             SDL_snprintf(failstr, failstrlen, "Out of memory");
-            return SDL_FALSE;
+            return NULL;
         }
         SDL_snprintf(parent_dir, slen, "%s", parent_fname);
         ptr = SDL_strrchr(parent_dir, '/');
@@ -934,6 +935,14 @@ static void handle_pp_error(Context *ctx)
 }
 
 
+static void handle_pp_bad(Context *ctx)
+{
+    IncludeState *state = ctx->include_stack;
+
+    failf(ctx, "unknown directive \"%s\"", state->token);
+}
+
+
 static void handle_pp_define(Context *ctx)
 {
     static const char space = ' ';
@@ -1265,6 +1274,7 @@ static SDL_bool replace_and_push_macro(Context *ctx, const Define *def, const De
     }
 
     state = ctx->include_stack;
+    state->expanding_macro = SDL_TRUE;
     while (lexer(state) != TOKEN_EOI) {
         SDL_bool wantorig = SDL_FALSE;
         const Define *arg = NULL;
@@ -1331,6 +1341,7 @@ static SDL_bool replace_and_push_macro(Context *ctx, const Define *def, const De
             goto replace_and_push_macro_failed;
         }
     }
+    state->expanding_macro = SDL_FALSE;
 
     final = buffer_flatten(buffer);
     if (!final) {
@@ -2106,6 +2117,9 @@ static inline const char *_preprocessor_nexttoken(Context *ctx, size_t *_len, To
             continue;  /* will return at top of loop. */
         } else if (token == TOKEN_PP_PRAGMA) {
             ctx->parsing_pragma = SDL_TRUE;
+        } else if (token == TOKEN_PP_BAD) {
+            handle_pp_bad(ctx);
+            continue;  /* will return at top of loop. */
         }
 
         /* !!! FIXME: was this meant to be an "else if"? */
@@ -2219,7 +2233,7 @@ const SDL_SHADER_PreprocessData *SDL_SHADER_Preprocess(const SDL_SHADER_Compiler
 
         prev_token = token;
     }
-    
+
     SDL_assert(token == TOKEN_EOI);
 
     total_bytes = buffer_size(buffer);
diff --git a/unit_tests/preprocessor/errors/unknown-directive b/unit_tests/preprocessor/errors/unknown-directive
new file mode 100644
index 0000000..5589088
--- /dev/null
+++ b/unit_tests/preprocessor/errors/unknown-directive
@@ -0,0 +1 @@
+#something
diff --git a/unit_tests/preprocessor/errors/unknown-directive.correct b/unit_tests/preprocessor/errors/unknown-directive.correct
new file mode 100644
index 0000000..1ef958f
--- /dev/null
+++ b/unit_tests/preprocessor/errors/unknown-directive.correct
@@ -0,0 +1 @@
+preprocessor/errors/unknown-directive:1: error: unknown directive "#something"
diff --git a/unit_tests/preprocessor/output/multiline-concat b/unit_tests/preprocessor/output/multiline-concat
new file mode 100644
index 0000000..32a8718
--- /dev/null
+++ b/unit_tests/preprocessor/output/multiline-concat
@@ -0,0 +1,3 @@
+#define A RI\
+GHT
+A
diff --git a/unit_tests/preprocessor/output/multiline-concat.correct b/unit_tests/preprocessor/output/multiline-concat.correct
new file mode 100644
index 0000000..db561ff
--- /dev/null
+++ b/unit_tests/preprocessor/output/multiline-concat.correct
@@ -0,0 +1 @@
+RIGHT
diff --git a/unit_tests/preprocessor/output/multiline-spaces b/unit_tests/preprocessor/output/multiline-spaces
new file mode 100644
index 0000000..b1639db
--- /dev/null
+++ b/unit_tests/preprocessor/output/multiline-spaces
@@ -0,0 +1,3 @@
+#define A LEFT \
+RIGHT
+A
diff --git a/unit_tests/preprocessor/output/multiline-spaces.correct b/unit_tests/preprocessor/output/multiline-spaces.correct
new file mode 100644
index 0000000..6ea09a5
--- /dev/null
+++ b/unit_tests/preprocessor/output/multiline-spaces.correct
@@ -0,0 +1 @@
+LEFT RIGHT