From b641c2a0db26127f3e379f8b92bc15feb500e242 Mon Sep 17 00:00:00 2001
From: Anonymous Maarten <[EMAIL REDACTED]>
Date: Fri, 4 Oct 2024 18:51:05 +0200
Subject: [PATCH] Refactor gendynapi.py with the final goal to make it
re-usable
---
src/dynapi/gendynapi.py | 468 ++++++++++++++++++----------------------
1 file changed, 213 insertions(+), 255 deletions(-)
diff --git a/src/dynapi/gendynapi.py b/src/dynapi/gendynapi.py
index 651174aa03483..dae72d241efb7 100755
--- a/src/dynapi/gendynapi.py
+++ b/src/dynapi/gendynapi.py
@@ -24,60 +24,68 @@
# output looks sane (git diff, it adds to existing files), and commit it.
# It keeps the dynamic API jump table operating correctly.
#
-# OS-specific API:
+# Platform-specific API:
# After running the script, you have to manually add #ifdef SDL_PLATFORM_WIN32
-# or similar around the function in 'SDL_dynapi_procs.h'
+# or similar around the function in 'SDL_dynapi_procs.h'.
#
import argparse
+import dataclasses
import json
+import logging
import os
-import pathlib
+from pathlib import Path
import pprint
import re
-SDL_ROOT = pathlib.Path(__file__).resolve().parents[2]
+SDL_ROOT = Path(__file__).resolve().parents[2]
SDL_INCLUDE_DIR = SDL_ROOT / "include/SDL3"
SDL_DYNAPI_PROCS_H = SDL_ROOT / "src/dynapi/SDL_dynapi_procs.h"
SDL_DYNAPI_OVERRIDES_H = SDL_ROOT / "src/dynapi/SDL_dynapi_overrides.h"
SDL_DYNAPI_SYM = SDL_ROOT / "src/dynapi/SDL_dynapi.sym"
-full_API = []
+RE_EXTERN_C = re.compile(r'.*extern[ "]*C[ "].*')
+RE_COMMENT_REMOVE_CONTENT = re.compile(r'\/\*.*\*/')
+RE_PARSING_FUNCTION = re.compile(r'(.*SDLCALL[^\(\)]*) ([a-zA-Z0-9_]+) *\((.*)\) *;.*')
+#eg:
+# void (SDLCALL *callback)(void*, int)
+# \1(\2)\3
+RE_PARSING_CALLBACK = re.compile(r'([^\(\)]*)\(([^\(\)]+)\)(.*)')
-def main():
- # Parse 'sdl_dynapi_procs_h' file to find existing functions
- existing_procs = find_existing_procs()
+logger = logging.getLogger(__name__)
- # Get list of SDL headers
- sdl_list_includes = get_header_list()
- reg_externC = re.compile(r'.*extern[ "]*C[ "].*')
- reg_comment_remove_content = re.compile(r'\/\*.*\*/')
- reg_parsing_function = re.compile(r'(.*SDLCALL[^\(\)]*) ([a-zA-Z0-9_]+) *\((.*)\) *;.*')
+@dataclasses.dataclass(frozen=True)
+class SdlProcedure:
+ retval: str
+ name: str
+ parameter: list[str]
+ parameter_name: list[str]
+ header: str
+ comment: str
- #eg:
- # void (SDLCALL *callback)(void*, int)
- # \1(\2)\3
- reg_parsing_callback = re.compile(r'([^\(\)]*)\(([^\(\)]+)\)(.*)')
+ @property
+ def variadic(self) -> bool:
+ return "..." in self.parameter
- for filename in sdl_list_includes:
- if args.debug:
- print("Parse header: %s" % filename)
- input = open(filename)
+def parse_header(header_path: Path) -> list[SdlProcedure]:
+ logger.debug("Parse header: %s", header_path)
- parsing_function = False
- current_func = ""
- parsing_comment = False
- current_comment = ""
+ header_procedures = []
- ignore_wiki_documentation = False
+ parsing_function = False
+ current_func = ""
+ parsing_comment = False
+ current_comment = ""
+ ignore_wiki_documentation = False
- for line in input:
+ with header_path.open() as f:
+ for line in f:
# Skip lines if we're in a wiki documentation block.
if ignore_wiki_documentation:
@@ -95,13 +103,13 @@ def main():
continue
# Discard "extern C" line
- match = reg_externC.match(line)
+ match = RE_EXTERN_C.match(line)
if match:
continue
# Remove one line comment // ...
- # eg: extern SDL_DECLSPEC SDL_hid_device * SDLCALL SDL_hid_open_path(const char *path, int bExclusive /* = false */);
- line = reg_comment_remove_content.sub('', line)
+ # eg: extern SDL_DECLSPEC SDL_hid_device * SDLCALL SDL_hid_open_path(const char *path, int bExclusive /* = false */)
+ line = RE_COMMENT_REMOVE_CONTENT.sub('', line)
# Get the comment block /* ... */ across several lines
match_start = "/*" in line
@@ -131,14 +139,14 @@ def main():
continue
# Start grabbing the new function
current_func = line.strip()
- parsing_function = True;
+ parsing_function = True
# If it contains ';', then the function is complete
if ";" not in current_func:
continue
# Got function/comment, reset vars
- parsing_function = False;
+ parsing_function = False
func = current_func
comment = current_comment
current_func = ""
@@ -146,47 +154,48 @@ def main():
# Discard if it doesn't contain 'SDLCALL'
if "SDLCALL" not in func:
- if args.debug:
- print(" Discard, doesn't have SDLCALL: " + func)
+ logger.debug(" Discard, doesn't have SDLCALL: %r", func)
continue
# Discard if it contains 'SDLMAIN_DECLSPEC' (these are not SDL symbols).
if "SDLMAIN_DECLSPEC" in func:
- if args.debug:
- print(" Discard, has SDLMAIN_DECLSPEC: " + func)
+ logger.debug(" Discard, has SDLMAIN_DECLSPEC: %r", func)
continue
- if args.debug:
- print(" Raw data: " + func);
+ logger.debug("Raw data: %r", func)
# Replace unusual stuff...
- func = func.replace(" SDL_PRINTF_VARARG_FUNC(1)", "");
- func = func.replace(" SDL_PRINTF_VARARG_FUNC(2)", "");
- func = func.replace(" SDL_PRINTF_VARARG_FUNC(3)", "");
- func = func.replace(" SDL_PRINTF_VARARG_FUNCV(1)", "");
- func = func.replace(" SDL_PRINTF_VARARG_FUNCV(2)", "");
- func = func.replace(" SDL_PRINTF_VARARG_FUNCV(3)", "");
- func = func.replace(" SDL_WPRINTF_VARARG_FUNC(3)", "");
- func = func.replace(" SDL_WPRINTF_VARARG_FUNCV(3)", "");
- func = func.replace(" SDL_SCANF_VARARG_FUNC(2)", "");
- func = func.replace(" SDL_SCANF_VARARG_FUNCV(2)", "");
- func = func.replace(" SDL_ANALYZER_NORETURN", "");
- func = func.replace(" SDL_MALLOC", "");
- func = func.replace(" SDL_ALLOC_SIZE2(1, 2)", "");
- func = func.replace(" SDL_ALLOC_SIZE(2)", "");
- func = re.sub(r" SDL_ACQUIRE\(.*\)", "", func);
- func = re.sub(r" SDL_ACQUIRE_SHARED\(.*\)", "", func);
- func = re.sub(r" SDL_TRY_ACQUIRE\(.*\)", "", func);
- func = re.sub(r" SDL_TRY_ACQUIRE_SHARED\(.*\)", "", func);
- func = re.sub(r" SDL_RELEASE\(.*\)", "", func);
- func = re.sub(r" SDL_RELEASE_SHARED\(.*\)", "", func);
- func = re.sub(r" SDL_RELEASE_GENERIC\(.*\)", "", func);
+ func = func.replace(" SDL_PRINTF_VARARG_FUNC(1)", "")
+ func = func.replace(" SDL_PRINTF_VARARG_FUNC(2)", "")
+ func = func.replace(" SDL_PRINTF_VARARG_FUNC(3)", "")
+ func = func.replace(" SDL_PRINTF_VARARG_FUNCV(1)", "")
+ func = func.replace(" SDL_PRINTF_VARARG_FUNCV(2)", "")
+ func = func.replace(" SDL_PRINTF_VARARG_FUNCV(3)", "")
+ func = func.replace(" SDL_WPRINTF_VARARG_FUNC(3)", "")
+ func = func.replace(" SDL_WPRINTF_VARARG_FUNCV(3)", "")
+ func = func.replace(" SDL_SCANF_VARARG_FUNC(2)", "")
+ func = func.replace(" SDL_SCANF_VARARG_FUNCV(2)", "")
+ func = func.replace(" SDL_ANALYZER_NORETURN", "")
+ func = func.replace(" SDL_MALLOC", "")
+ func = func.replace(" SDL_ALLOC_SIZE2(1, 2)", "")
+ func = func.replace(" SDL_ALLOC_SIZE(2)", "")
+ func = re.sub(r" SDL_ACQUIRE\(.*\)", "", func)
+ func = re.sub(r" SDL_ACQUIRE_SHARED\(.*\)", "", func)
+ func = re.sub(r" SDL_TRY_ACQUIRE\(.*\)", "", func)
+ func = re.sub(r" SDL_TRY_ACQUIRE_SHARED\(.*\)", "", func)
+ func = re.sub(r" SDL_RELEASE\(.*\)", "", func)
+ func = re.sub(r" SDL_RELEASE_SHARED\(.*\)", "", func)
+ func = re.sub(r" SDL_RELEASE_GENERIC\(.*\)", "", func)
+ func = re.sub(r"([ (),])(SDL_IN_BYTECAP\([^)]*\))", r"\1", func)
+ func = re.sub(r"([ (),])(SDL_OUT_BYTECAP\([^)]*\))", r"\1", func)
+ func = re.sub(r"([ (),])(SDL_INOUT_Z_CAP\([^)]*\))", r"\1", func)
+ func = re.sub(r"([ (),])(SDL_OUT_Z_CAP\([^)]*\))", r"\1", func)
# Should be a valid function here
- match = reg_parsing_function.match(func)
+ match = RE_PARSING_FUNCTION.match(func)
if not match:
- print("Cannot parse: "+ func)
- exit(-1)
+ logger.error("Cannot parse: %s", func)
+ raise ValueError(func)
func_ret = match.group(1)
func_name = match.group(2)
@@ -198,11 +207,8 @@ def main():
func_ret = func_ret.replace('extern', ' ')
func_ret = func_ret.replace('SDLCALL', ' ')
func_ret = func_ret.replace('SDL_DECLSPEC', ' ')
+ func_ret, _ = re.subn('([ ]{2,})', ' ', func_ret)
# Remove trailing spaces in front of '*'
- tmp = ""
- while func_ret != tmp:
- tmp = func_ret
- func_ret = func_ret.replace(' ', ' ')
func_ret = func_ret.replace(' *', '*')
func_ret = func_ret.strip()
@@ -246,10 +252,10 @@ def main():
# parameter is a callback
if '(' in t:
- match = reg_parsing_callback.match(t)
+ match = RE_PARSING_CALLBACK.match(t)
if not match:
- print("cannot parse callback: " + t);
- exit(-1)
+ logger.error("cannot parse callback: %s", t)
+ raise ValueError(t)
a = match.group(1).strip()
b = match.group(2).strip()
c = match.group(3).strip()
@@ -257,7 +263,7 @@ def main():
try:
(param_type, param_name) = b.rsplit('*', 1)
except:
- param_type = t;
+ param_type = t
param_name = "param_name_not_specified"
# bug rsplit ??
@@ -281,7 +287,7 @@ def main():
try:
(param_type, param_name) = t.rsplit('*', 1)
except:
- param_type = t;
+ param_type = t
param_name = "param_name_not_specified"
# bug rsplit ??
@@ -305,7 +311,7 @@ def main():
try:
(param_type, param_name) = t.rsplit(' ', 1)
except:
- param_type = t;
+ param_type = t
param_name = "param_name_not_specified"
val = param_type.strip() + " REWRITE_NAME"
@@ -317,268 +323,220 @@ def main():
func_param_type.append(val)
func_param_name.append(param_name.strip())
- new_proc = {}
- # Return value type
- new_proc['retval'] = func_ret
- # List of parameters (type + anonymized param name 'REWRITE_NAME')
- new_proc['parameter'] = func_param_type
- # Real parameter name, or 'param_name_not_specified'
- new_proc['parameter_name'] = func_param_name
- # Function name
- new_proc['name'] = func_name
- # Header file
- new_proc['header'] = os.path.basename(filename)
- # Function comment
- new_proc['comment'] = comment
+ new_proc = SdlProcedure(
+ retval=func_ret, # Return value type
+ name=func_name, # Function name
+ comment=comment, # Function comment
+ header=header_path.name, # Header file
+ parameter=func_param_type, # List of parameters (type + anonymized param name 'REWRITE_NAME')
+ parameter_name=func_param_name, # Real parameter name, or 'param_name_not_specified'
+ )
- full_API.append(new_proc)
+ header_procedures.append(new_proc)
- if args.debug:
- pprint.pprint(new_proc);
- print("\n")
+ if logger.getEffectiveLevel() <= logging.DEBUG:
+ logger.debug("%s", pprint.pformat(new_proc))
- if func_name not in existing_procs:
- print("NEW " + func)
- add_dyn_api(new_proc)
+ return header_procedures
- # For-End line in input
- input.close()
- # For-End parsing all files of sdl_list_includes
+# Dump API into a json file
+def full_API_json(path: Path, procedures: list[SdlProcedure]):
+ with path.open('w', newline='') as f:
+ json.dump([dataclasses.asdict(proc) for proc in procedures], f, indent=4, sort_keys=True)
+ logger.info("dump API to '%s'", path)
- # Dump API into a json file
- full_API_json()
- # Check comment formatting
- check_comment();
+class CallOnce:
+ def __init__(self, cb):
+ self._cb = cb
+ self._called = False
+ def __call__(self, *args, **kwargs):
+ if self._called:
+ return
+ self._called = True
+ self._cb(*args, **kwargs)
-# Dump API into a json file
-def full_API_json():
- if args.dump:
- filename = 'sdl.json'
- with open(filename, 'w', newline='') as f:
- json.dump(full_API, f, indent=4, sort_keys=True)
- print("dump API to '%s'" % filename);
# Check public function comments are correct
-def check_comment_header():
- if not check_comment_header.done:
- check_comment_header.done = True
- print("")
- print("Please fix following warning(s):")
- print("-------------------------------")
+def print_check_comment_header():
+ logger.warning("")
+ logger.warning("Please fix following warning(s):")
+ logger.warning("--------------------------------")
-def check_comment():
+def check_documentations(procedures: list[SdlProcedure]) -> None:
- check_comment_header.done = False
+ check_comment_header = CallOnce(print_check_comment_header)
- # Check \param
- for i in full_API:
- comment = i['comment']
- name = i['name']
- retval = i['retval']
- header = i['header']
+ warning_header_printed = False
- expected = len(i['parameter'])
+ # Check \param
+ for proc in procedures:
+ expected = len(proc.parameter)
if expected == 1:
- if i['parameter'][0] == 'void':
- expected = 0;
- count = comment.count("\\param")
+ if proc.parameter[0] == 'void':
+ expected = 0
+ count = proc.comment.count("\\param")
if count != expected:
# skip SDL_stdinc.h
- if header != 'SDL_stdinc.h':
+ if proc.header != 'SDL_stdinc.h':
# Warning mismatch \param and function prototype
check_comment_header()
- print(" In file %s: function %s() has %d '\\param' but expected %d" % (header, name, count, expected));
+ logger.warning(" In file %s: function %s() has %d '\\param' but expected %d", proc.header, proc.name, count, expected)
# Warning check \param uses the correct parameter name
# skip SDL_stdinc.h
- if header != 'SDL_stdinc.h':
- parameter_name = i['parameter_name']
- for n in parameter_name:
- if n != "" and "\\param " + n not in comment and "\\param[out] " + n not in comment:
+ if proc.header != 'SDL_stdinc.h':
+ for n in proc.parameter_name:
+ if n != "" and "\\param " + n not in proc.comment and "\\param[out] " + n not in proc.comment:
check_comment_header()
- print(" In file %s: function %s() missing '\\param %s'" % (header, name, n));
-
+ logger.warning(" In file %s: function %s() missing '\\param %s'", proc.header, proc.name, n)
# Check \returns
- for i in full_API:
- comment = i['comment']
- name = i['name']
- retval = i['retval']
- header = i['header']
-
+ for proc in procedures:
expected = 1
- if retval == 'void':
- expected = 0;
+ if proc.retval == 'void':
+ expected = 0
- count = comment.count("\\returns")
+ count = proc.comment.count("\\returns")
if count != expected:
# skip SDL_stdinc.h
- if header != 'SDL_stdinc.h':
+ if proc.header != 'SDL_stdinc.h':
# Warning mismatch \param and function prototype
check_comment_header()
- print(" In file %s: function %s() has %d '\\returns' but expected %d" % (header, name, count, expected));
+ logger.warning(" In file %s: function %s() has %d '\\returns' but expected %d" % (proc.header, proc.name, count, expected))
# Check \since
- for i in full_API:
- comment = i['comment']
- name = i['name']
- retval = i['retval']
- header = i['header']
-
+ for proc in procedures:
expected = 1
- count = comment.count("\\since")
+ count = proc.comment.count("\\since")
if count != expected:
# skip SDL_stdinc.h
- if header != 'SDL_stdinc.h':
+ if proc.header != 'SDL_stdinc.h':
# Warning mismatch \param and function prototype
check_comment_header()
- print(" In file %s: function %s() has %d '\\since' but expected %d" % (header, name, count, expected));
-
+ logger.warning(" In file %s: function %s() has %d '\\since' but expected %d" % (proc.header, proc.name, count, expected))
# Parse 'sdl_dynapi_procs_h' file to find existing functions
-def find_existing_procs():
+def find_existing_proc_names() -> list[str]:
reg = re.compile(r'SDL_DYNAPI_PROC\([^,]*,([^,]*),.*\)')
ret = []
- input = open(SDL_DYNAPI_PROCS_H)
-
- for line in input:
- match = reg.match(line)
- if not match:
- continue
- existing_func = match.group(1)
- ret.append(existing_func);
- # print(existing_func)
- input.close()
+ with SDL_DYNAPI_PROCS_H.open() as f:
+ for line in f:
+ match = reg.match(line)
+ if not match:
+ continue
+ existing_func = match.group(1)
+ ret.append(existing_func)
return ret
# Get list of SDL headers
-def get_header_list():
- reg = re.compile(r'^.*\.h$')
+def get_header_list() -> list[Path]:
ret = []
- tmp = os.listdir(SDL_INCLUDE_DIR)
- for f in tmp:
+ for f in SDL_INCLUDE_DIR.iterdir():
# Only *.h files
- match = reg.match(f)
- if not match:
- if args.debug:
- print("Skip %s" % f)
- continue
- ret.append(SDL_INCLUDE_DIR / f)
+ if f.is_file() and f.suffix == ".h":
+ ret.append(f)
+ else:
+ logger.debug("Skip %s", f)
return ret
# Write the new API in files: _procs.h _overrivides.h and .sym
-def add_dyn_api(proc):
- func_name = proc['name']
- func_ret = proc['retval']
- func_argtype = proc['parameter']
-
-
- # File: SDL_dynapi_procs.h
- #
- # Add at last
- # SDL_DYNAPI_PROC(SDL_EGLConfig,SDL_EGL_GetCurrentConfig,(void),(),return)
- f = open(SDL_DYNAPI_PROCS_H, "a", newline="")
- dyn_proc = "SDL_DYNAPI_PROC(" + func_ret + "," + func_name + ",("
-
- i = ord('a')
- remove_last = False
- for argtype in func_argtype:
-
- # Special case, void has no parameter name
- if argtype == "void":
- dyn_proc += "void"
- continue
-
- # Var name: a, b, c, ...
- varname = chr(i)
- i += 1
-
- tmp = argtype.replace("REWRITE_NAME", varname)
- dyn_proc += tmp + ", "
- remove_last = True
-
- # remove last 2 char ', '
- if remove_last:
- dyn_proc = dyn_proc[:-1]
- dyn_proc = dyn_proc[:-1]
-
- dyn_proc += "),("
-
- i = ord('a')
- remove_last = False
- for argtype in func_argtype:
-
+def add_dyn_api(proc: SdlProcedure) -> None:
+ decl_args: list[str] = []
+ call_args = []
+ for i, argtype in enumerate(proc.parameter):
# Special case, void has no parameter name
if argtype == "void":
- continue
-
- # Special case, '...' has no parameter name
- if argtype == "...":
+ assert len(decl_args) == 0
+ assert len(proc.parameter) == 1
+ decl_args.append("void")
continue
# Var name: a, b, c, ...
- varname = chr(i)
- i += 1
+ varname = chr(ord('a') + i)
- dyn_proc += varname + ","
- remove_last = True
+ decl_args.append(argtype.replace("REWRITE_NAME", varname))
+ if argtype != "...":
+ call_args.append(varname)
- # remove last char ','
- if remove_last:
- dyn_proc = dyn_proc[:-1]
+ macro_args = (
+ proc.retval,
+ proc.name,
+ "({})".format(",".join(decl_args)),
+ "({})".format(",".join(call_args)),
+ "" if proc.retval == "void" else "return",
+ )
- dyn_proc += "),"
-
- if func_ret != "void":
- dyn_proc += "return"
- dyn_proc += ")"
- f.write(dyn_proc + "\n")
- f.close()
+ # File: SDL_dynapi_procs.h
+ #
+ # Add at last
+ # SDL_DYNAPI_PROC(SDL_EGLConfig,SDL_EGL_GetCurrentConfig,(void),(),return)
+ with SDL_DYNAPI_PROCS_H.open("a", newline="") as f:
+ if proc.variadic:
+ f.write("#ifndef SDL_DYNAPI_PROC_NO_VARARGS\n")
+ f.write(f"SDL_DYNAPI_PROC({','.join(macro_args)})\n")
+ if proc.variadic:
+ f.write("#endif\n")
# File: SDL_dynapi_overrides.h
#
# Add at last
# "#define SDL_DelayNS SDL_DelayNS_REAL
f = open(SDL_DYNAPI_OVERRIDES_H, "a", newline="")
- f.write("#define " + func_name + " " + func_name + "_REAL\n")
+ f.write(f"#define {proc.name} {proc.name}_REAL\n")
f.close()
# File: SDL_dynapi.sym
#
# Add before "extra symbols go here" line
- input = open(SDL_DYNAPI_SYM)
- new_input = []
- for line in input:
- if "extra symbols go here" in line:
- new_input.append(" " + func_name + ";\n")
- new_input.append(line)
- input.close()
- f = open(SDL_DYNAPI_SYM, 'w', newline='')
- for line in new_input:
- f.write(line)
- f.close()
+ with SDL_DYNAPI_SYM.open() as f:
+ new_input = []
+ for line in f:
+ if "extra symbols go here" in line:
+ new_input.append(f" {proc.name};\n")
+ new_input.append(line)
+ with SDL_DYNAPI_SYM.open('w', newline='') as f:
+ for line in new_input:
+ f.write(line)
-if __name__ == '__main__':
+def main():
parser = argparse.ArgumentParser()
- parser.add_argument('--dump', help='output all SDL API into a .json file', action='store_true')
- parser.add_argument('--debug', help='add debug traces', action='store_true')
+ parser.set_defaults(loglevel=logging.INFO)
+ parser.add_argument('--dump', nargs='?', default=None, const="sdl.json", metavar="JSON", help='output all SDL API into a .json file')
+ parser.add_argument('--debug', action='store_const', const=logging.DEBUG, dest="loglevel", help='add debug traces')
args = parser.parse_args()
- try:
- main()
- except Exception as e:
- print(e)
- exit(-1)
+ logging.basicConfig(level=args.loglevel, format='[%(levelname)s] %(message)s')
- print("done!")
- exit(0)
+ # Get list of SDL headers
+ sdl_list_includes = get_header_list()
+ procedures = []
+ for filename in sdl_list_includes:
+ header_procedures = parse_header(filename)
+ procedures.extend(header_procedures)
+ # Parse 'sdl_dynapi_procs_h' file to find existing functions
+ existing_proc_names = find_existing_proc_names()
+ for procedure in procedures:
+ if procedure.name not in existing_proc_names:
+ logger.info("NEW %s", procedure.name)
+ add_dyn_api(procedure)
+
+ if args.dump:
+ # Dump API into a json file
+ full_API_json(path=Path(args.dump), procedures=procedures)
+
+ # Check comment formatting
+ check_documentations(procedures)
+
+
+if __name__ == '__main__':
+ raise SystemExit(main())