From 20a6193eaa3166f420071028d4c0081ed1f21cbe Mon Sep 17 00:00:00 2001
From: Sylvain <[EMAIL REDACTED]>
Date: Wed, 7 Aug 2024 09:55:08 +0200
Subject: [PATCH] Run test suites and test cases in non-linear order (see
libsdl-org#9303)
---
include/SDL3/SDL_test_harness.h | 3 +-
src/test/SDL_test_harness.c | 96 +++++++++++++++++++++++++++++++--
test/testautomation.c | 16 +++++-
3 files changed, 107 insertions(+), 8 deletions(-)
diff --git a/include/SDL3/SDL_test_harness.h b/include/SDL3/SDL_test_harness.h
index 263071fbd3928..70adc3585ae5c 100644
--- a/include/SDL3/SDL_test_harness.h
+++ b/include/SDL3/SDL_test_harness.h
@@ -117,10 +117,11 @@ char *SDLTest_GenerateRunSeed(const int length);
* \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
* \param filter Filter specification. NULL disables. Case sensitive.
* \param testIterations Number of iterations to run each test case.
+ * \param randomOrder allow to run suites and tests in random order when there is no filter
*
* \returns the test run result: 0 when all tests passed, 1 if any tests failed.
*/
-int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations);
+int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations, SDL_bool randomOrder);
/* Ends C function definitions when using C++ */
diff --git a/src/test/SDL_test_harness.c b/src/test/SDL_test_harness.c
index e306e4630d4de..f91d4903e1360 100644
--- a/src/test/SDL_test_harness.c
+++ b/src/test/SDL_test_harness.c
@@ -365,10 +365,11 @@ static float GetClock(void)
* \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
* \param filter Filter specification. NULL disables. Case sensitive.
* \param testIterations Number of iterations to run each test case.
+ * \param randomOrder allow to run suites and tests in random order when there is no filter
*
* \returns Test run result; 0 when all tests passed, 1 if any tests failed.
*/
-int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
+int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations, SDL_bool randomOrder)
{
int totalNumberOfTests = 0;
int failedNumberOfTests = 0;
@@ -404,6 +405,9 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
int countSum = 0;
const SDLTest_TestCaseReference **failedTests;
char generatedSeed[16 + 1];
+ int nbSuites = 0;
+ int i = 0;
+ int *arraySuites = NULL;
/* Sanitize test iterations */
if (testIterations < 1) {
@@ -509,11 +513,59 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
SDL_free((void *)failedTests);
return 2;
}
+
+ randomOrder = SDL_FALSE;
+ }
+
+ /* Number of test suites */
+ while (testSuites[nbSuites]) {
+ nbSuites++;
+ }
+
+ arraySuites = SDL_malloc(nbSuites * sizeof(int));
+ if (!arraySuites) {
+ return SDL_OutOfMemory();
+ }
+ for (i = 0; i < nbSuites; i++) {
+ arraySuites[i] = i;
+ }
+
+ /* Mix the list of suites to run them in random order */
+ {
+ /* Exclude last test "subsystemsTestSuite" which is said to interfer with other tests */
+ nbSuites--;
+
+ if (userExecKey != 0) {
+ execKey = userExecKey;
+ } else {
+ /* dummy values to have random numbers working */
+ execKey = SDLTest_GenerateExecKey(runSeed, "random testSuites", "initialisation", 1);
+ }
+
+ /* Initialize fuzzer */
+ SDLTest_FuzzerInit(execKey);
+
+ i = 100;
+ while (i--) {
+ int a, b;
+ int tmp;
+ a = SDLTest_RandomIntegerInRange(0, nbSuites - 1);
+ b = SDLTest_RandomIntegerInRange(0, nbSuites - 1);
+ /* Swap */
+ if (randomOrder) {
+ tmp = arraySuites[b];
+ arraySuites[b] = arraySuites[a];
+ arraySuites[a] = tmp;
+ }
+ }
+
+ /* re-add last lest */
+ nbSuites++;
}
/* Loop over all suites */
- suiteCounter = 0;
- while (testSuites[suiteCounter]) {
+ for (i = 0; i < nbSuites; i++) {
+ suiteCounter = arraySuites[i];;
testSuite = testSuites[suiteCounter];
currentSuiteName = (testSuite->name ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT);
suiteCounter++;
@@ -527,6 +579,36 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
currentSuiteName);
} else {
+ int nbTestCases = 0;
+ int *arrayTestCases;
+ int j;
+ while (testSuite->testCases[nbTestCases]) {
+ nbTestCases++;
+ }
+
+ arrayTestCases = SDL_malloc(nbTestCases * sizeof(int));
+ if (!arrayTestCases) {
+ return SDL_OutOfMemory();
+ }
+ for (j = 0; j < nbTestCases; j++) {
+ arrayTestCases[j] = j;
+ }
+
+ /* Mix the list of testCases to run them in random order */
+ j = 100;
+ while (j--) {
+ int a, b;
+ int tmp;
+ a = SDLTest_RandomIntegerInRange(0, nbTestCases - 1);
+ b = SDLTest_RandomIntegerInRange(0, nbTestCases - 1);
+ /* Swap */
+ if (randomOrder) {
+ tmp = arrayTestCases[b];
+ arrayTestCases[b] = arrayTestCases[a];
+ arrayTestCases[a] = tmp;
+ }
+ }
+
/* Reset per-suite counters */
testFailedCount = 0;
testPassedCount = 0;
@@ -541,8 +623,8 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
currentSuiteName);
/* Loop over all test cases */
- testCounter = 0;
- while (testSuite->testCases[testCounter]) {
+ for (j = 0; j < nbTestCases; j++) {
+ testCounter = arrayTestCases[j];
testCase = testSuite->testCases[testCounter];
currentTestName = (testCase->name ? testCase->name : SDLTEST_INVALID_NAME_FORMAT);
testCounter++;
@@ -657,9 +739,13 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, COLOR_RED "Failed" COLOR_END);
}
+
+ SDL_free(arrayTestCases);
}
}
+ SDL_free(arraySuites);
+
/* Take time - run end */
runEndSeconds = GetClock();
runtime = runEndSeconds - runStartSeconds;
diff --git a/test/testautomation.c b/test/testautomation.c
index a8de28bf911cf..42dfeef58dd5e 100644
--- a/test/testautomation.c
+++ b/test/testautomation.c
@@ -75,6 +75,7 @@ int main(int argc, char *argv[])
int i, done;
SDL_Event event;
int list = 0;
+ SDL_bool randomOrder = SDL_FALSE;
/* Initialize test framework */
state = SDLTest_CommonCreateState(argv, SDL_INIT_VIDEO | SDL_INIT_AUDIO);
@@ -118,10 +119,21 @@ int main(int argc, char *argv[])
} else if (SDL_strcasecmp(argv[i], "--list") == 0) {
consumed = 1;
list = 1;
+ } else if (SDL_strcasecmp(argv[i], "--random-order") == 0) {
+ consumed = 1;
+ randomOrder = SDL_TRUE;
}
+
}
if (consumed < 0) {
- static const char *options[] = { "[--iterations #]", "[--execKey #]", "[--seed string]", "[--filter suite_name|test_name]", "[--list]", NULL };
+ static const char *options[] = {
+ "[--iterations #]",
+ "[--execKey #]",
+ "[--seed string]",
+ "[--filter suite_name|test_name]",
+ "[--list]",
+ "[--random-order]",
+ NULL };
SDLTest_CommonLogUsage(state, argv[0], options);
quit(1);
}
@@ -157,7 +169,7 @@ int main(int argc, char *argv[])
}
/* Call Harness */
- result = SDLTest_RunSuites(testSuites, userRunSeed, userExecKey, filter, testIterations);
+ result = SDLTest_RunSuites(testSuites, userRunSeed, userExecKey, filter, testIterations, randomOrder);
/* Empty event queue */
done = 0;