GCC Code Coverage Report

Directory: ../
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 63.8% 201 / 31 / 346
Functions: 85.5% 65 / 3 / 79
Branches: 29.1% 344 / 94 / 1277
test/TestRunner.cpp
Line Branch Exec Source
1 // Copyright (c) 2021-2026 ChilliBits. All rights reserved.
2
3 #include <algorithm>
4 #include <string>
5 #include <unordered_set>
6 #include <vector>
7
8 #include <gtest/gtest.h>
9
10 #include <SourceFile.h>
11 #include <driver/Driver.h>
12 #include <exception/CompilerError.h>
13 #include <exception/LexerError.h>
14 #include <exception/LinkerError.h>
15 #include <exception/ParserError.h>
16 #include <exception/SemanticError.h>
17 #include <global/GlobalResourceManager.h>
18 #include <global/TypeRegistry.h>
19 #include <llvm/IR/Module.h>
20 #include <llvm/TargetParser/Host.h>
21 #include <llvm/TargetParser/Triple.h>
22 #include <symboltablebuilder/Scope.h>
23 #include <symboltablebuilder/SymbolTable.h>
24 #include <typechecker/FunctionManager.h>
25 #include <typechecker/InterfaceManager.h>
26 #include <typechecker/StructManager.h>
27 #include <util/FileUtil.h>
28 #include <util/SystemUtil.h>
29
30 #include "driver/TestDriver.h"
31 #include "util/BootstrapUtil.h"
32 #include "util/TestUtil.h"
33
34 using namespace spice::compiler;
35
36 namespace spice::testing {
37
38 extern TestDriverCliOptions testDriverCliOptions;
39
40 namespace {
41
42 /**
43 * Re-run the IR generator for the given source file, discarding the module that was built for it before.
44 *
45 * Since the IR generator emits parts of the IR depending on the selected opt level, a module can only be used to check
46 * the dump of the opt level it was generated for.
47 *
48 * @param sourceFile Source file to re-generate the IR for
49 */
50 80 void reGenerateIR(SourceFile *sourceFile) {
51 // Rewind the stage marker, so that the IR generator does not consider this file as already done
52 80 sourceFile->previousStage = DEP_GRAPH_VISUALIZER;
53 80 sourceFile->runIRGenerator();
54 80 }
55
56 /**
57 * Re-run the IR generator for the given source file and all its (transitive) dependencies, dependencies first.
58 *
59 * This is only required for LTO builds, where the modules of all source files are merged into the LTO module, which is
60 * the one that gets dumped. Without LTO, only the module of the main source file ends up in the dump, so re-generating
61 * the dependencies would be wasted work.
62 *
63 * @param sourceFile Source file to re-generate the IR for
64 * @param visitedSet Already visited source files (guards against cycles, caused by circular imports)
65 */
66 2 void reGenerateIRForLTO(SourceFile *sourceFile, std::unordered_set<const SourceFile *> &visitedSet) { // NOLINT(misc-no-recursion)
67
1/2
✗ Branch 3 → 4 not taken.
✓ Branch 3 → 5 taken 2 times.
 2 if (!visitedSet.insert(sourceFile).second)
68 ✗ return;
69
70 // Re-generate the dependencies first, mirroring the order of SourceFile::runBackEnd()
71
5/8
✓ Branch 5 → 6 taken 2 times.
✗ Branch 5 → 18 not taken.
✓ Branch 6 → 7 taken 2 times.
✗ Branch 6 → 18 not taken.
✓ Branch 7 → 8 taken 2 times.
✗ Branch 7 → 18 not taken.
✓ Branch 13 → 9 taken 1 time.
✓ Branch 13 → 14 taken 2 times.
 3 for (SourceFile *dependency : sourceFile->dependencies | std::views::values)
72
1/2
✓ Branch 10 → 11 taken 1 time.
✗ Branch 10 → 18 not taken.
 1 reGenerateIRForLTO(dependency, visitedSet);
73
74 2 reGenerateIR(sourceFile);
75
76 // The optimizer stages of the main source file are driven by the test runner itself. All other files run their
77 // pre-link optimization as part of their own back end, so it has to be repeated here.
78
2/2
✓ Branch 15 → 16 taken 1 time.
✓ Branch 15 → 17 taken 1 time.
 2 if (!sourceFile->isMainFile)
79 1 sourceFile->runPreLinkIROptimizer();
80 }
81
82 } // namespace
83
84 1342 static void execTestCase(const TestCase &testCase) {
85 // Check if test is disabled
86
3/4
✓ Branch 2 → 3 taken 1342 times.
✗ Branch 2 → 741 not taken.
✓ Branch 3 → 4 taken 27 times.
✓ Branch 3 → 10 taken 1315 times.
 1342 if (TestUtil::isDisabled(testCase))
87
3/6
✓ Branch 4 → 5 taken 27 times.
✗ Branch 4 → 388 not taken.
✓ Branch 5 → 6 taken 27 times.
✗ Branch 5 → 385 not taken.
✓ Branch 6 → 7 taken 27 times.
✗ Branch 6 → 383 not taken.
 27 GTEST_SKIP();
88
89 // Create fake cli options
90
2/4
✓ Branch 10 → 11 taken 1315 times.
✗ Branch 10 → 391 not taken.
✓ Branch 11 → 12 taken 1315 times.
✗ Branch 11 → 389 not taken.
 1315 const std::filesystem::path mainSourceFilePath = testCase.testPath / REF_NAME_SOURCE;
91 1315 CliOptions cliOptions = {
92 /* mainSourceFile= */ mainSourceFilePath,
93 /* targetTriple= */ {},
94 − /* targetArch= */ TARGET_UNKNOWN, // GCOV_EXCL_LINE - coverage tool bug
95 ✗ /* targetVendor= */ TARGET_UNKNOWN,
96 ✗ /* targetOs= */ TARGET_UNKNOWN,
97 /* isNativeTarget= */ true,
98 /* useCPUFeatures*/ false, // Disabled because it makes the refs differ on different machines
99 /* execute= */ false, // If we set this to 'true', the compiler will not emit object files
100 /* cacheDir= */ "./cache",
101 /* outputDir= */ "./",
102 /* outputPath= */ "",
103 /* buildMode= */ BuildMode::DEBUG,
104 /* outputContainer= */ OutputContainer::EXECUTABLE,
105 // Compile serially: the test cases are already spread over processes by gtest-parallel, so a parallel back end
106 // would only oversubscribe the machine, and the reference outputs stay strictly deterministic this way.
107 /* compileJobCount= */ 1,
108 /* ignoreCache */ true,
109 ✗ /* llvmArgs= */ "",
110 /* printDebugOutput= */ false,
111 CliOptions::DumpSettings{
112 /* dumpAST= */ false,
113 /* dumpSymbolTables= */ false,
114 /* dumpTypes= */ false,
115 /* dumpCacheStats= */ false,
116 /* dumpDependencyGraph= */ false,
117 /* dumpIR= */ false,
118 /* dumpAssembly= */ false,
119 /* dumpObjectFile= */ false,
120 /* dumpToFiles= */ false,
121 /* abortAfterDump */ false,
122 },
123 /* namesForIRValues= */ true,
124 /* useLifetimeMarkers= */ false,
125 /* useTBAAMetadata */ false,
126 /* optLevel= */ OptLevel::O0,
127 /* useLTO= */ false,
128 /* backend= */ Backend::LLVM,
129
2/4
✓ Branch 31 → 32 taken 1315 times.
✗ Branch 31 → 400 not taken.
✓ Branch 32 → 33 taken 1315 times.
✗ Branch 32 → 398 not taken.
 2630 /* noEntryFct= */ exists(testCase.testPath / CTL_RUN_BUILTIN_TESTS),
130
2/4
✓ Branch 34 → 35 taken 1315 times.
✗ Branch 34 → 394 not taken.
✓ Branch 35 → 36 taken 1315 times.
✗ Branch 35 → 392 not taken.
 2630 /* generateTestMain= */ exists(testCase.testPath / CTL_RUN_BUILTIN_TESTS),
131 /* staticLinking= */ false,
132 CliOptions::InstrumentationSettings{
133 /* debugInfoLevel= */ DebugInfoLevel::NONE,
134 1315 /* codeCoverage= */ testDriverCliOptions.enableCoverage,
135 /* sanitizer= */ Sanitizer::NONE,
136 },
137 /* disableVerifier= */ false,
138 /* testMode= */ true,
139 /* comparableOutput= */ true,
140 /* buildVars= */ {},
141
20/58
✓ Branch 13 → 14 taken 1315 times.
✗ Branch 13 → 739 not taken.
✓ Branch 17 → 18 taken 1315 times.
✗ Branch 17 → 434 not taken.
✓ Branch 20 → 21 taken 1315 times.
✗ Branch 20 → 428 not taken.
✓ Branch 23 → 24 taken 1315 times.
✗ Branch 23 → 422 not taken.
✓ Branch 24 → 25 taken 1315 times.
✗ Branch 24 → 419 not taken.
✓ Branch 25 → 26 taken 1315 times.
✗ Branch 25 → 416 not taken.
✓ Branch 26 → 27 taken 1315 times.
✗ Branch 26 → 413 not taken.
✓ Branch 29 → 30 taken 1315 times.
✗ Branch 29 → 407 not taken.
✓ Branch 30 → 31 taken 1315 times.
✗ Branch 30 → 402 not taken.
✓ Branch 33 → 34 taken 1315 times.
✗ Branch 33 → 396 not taken.
✗ Branch 37 → 38 not taken.
✓ Branch 37 → 39 taken 1315 times.
✗ Branch 43 → 44 not taken.
✓ Branch 43 → 45 taken 1315 times.
✗ Branch 46 → 47 not taken.
✓ Branch 46 → 48 taken 1315 times.
✗ Branch 48 → 49 not taken.
✓ Branch 48 → 50 taken 1315 times.
✗ Branch 50 → 51 not taken.
✓ Branch 50 → 52 taken 1315 times.
✗ Branch 52 → 53 not taken.
✓ Branch 52 → 54 taken 1315 times.
✗ Branch 55 → 56 not taken.
✓ Branch 55 → 57 taken 1315 times.
✗ Branch 58 → 59 not taken.
✓ Branch 58 → 60 taken 1315 times.
✗ Branch 61 → 62 not taken.
✓ Branch 61 → 63 taken 1315 times.
✗ Branch 63 → 64 not taken.
✓ Branch 63 → 65 taken 1315 times.
✗ Branch 404 → 405 not taken.
✗ Branch 404 → 406 not taken.
✗ Branch 410 → 411 not taken.
✗ Branch 410 → 412 not taken.
✗ Branch 413 → 414 not taken.
✗ Branch 413 → 415 not taken.
✗ Branch 416 → 417 not taken.
✗ Branch 416 → 418 not taken.
✗ Branch 419 → 420 not taken.
✗ Branch 419 → 421 not taken.
✗ Branch 425 → 426 not taken.
✗ Branch 425 → 427 not taken.
✗ Branch 431 → 432 not taken.
✗ Branch 431 → 433 not taken.
✗ Branch 437 → 438 not taken.
✗ Branch 437 → 439 not taken.
✗ Branch 440 → 441 not taken.
✗ Branch 440 → 442 not taken.
 15780 };
142 static_assert(sizeof(CliOptions::DumpSettings) == 10, "CliOptions::DumpSettings struct size changed");
143 static_assert(sizeof(CliOptions::InstrumentationSettings) == 3, "CliOptions::InstrumentationSettings struct size changed");
144 #if defined(__clang__) && defined(__apple_build_version__)
145 // some std types for Apple Clang are smaller than for GCC and Clang
146 static_assert(sizeof(CliOptions) == 320, "CliOptions struct size changed");
147 #else
148 static_assert(sizeof(CliOptions) == 448, "CliOptions struct size changed");
149 #endif
150
151 // Parse test args
152
1/2
✓ Branch 67 → 68 taken 1315 times.
✗ Branch 67 → 443 not taken.
 2630 std::vector<std::string> args = {"spice", "build"};
153
1/2
✓ Branch 69 → 70 taken 1315 times.
✗ Branch 69 → 735 not taken.
 1315 TestUtil::parseTestArgs(cliOptions.mainSourceFile, args);
154
2/4
✓ Branch 70 → 71 taken 1315 times.
✗ Branch 70 → 448 not taken.
✓ Branch 71 → 72 taken 1315 times.
✗ Branch 71 → 446 not taken.
 1315 args.push_back(mainSourceFilePath.string());
155
156 1315 bool explicitlySelectedTarget = false;
157 1315 std::vector<const char *> argv;
158
1/2
✓ Branch 74 → 75 taken 1315 times.
✗ Branch 74 → 733 not taken.
 1315 argv.reserve(args.size());
159
2/2
✓ Branch 93 → 77 taken 4042 times.
✓ Branch 93 → 94 taken 1315 times.
 6672 for (const std::string &arg : args) {
160
2/2
✓ Branch 80 → 81 taken 10 times.
✓ Branch 80 → 82 taken 4032 times.
 4042 if (arg.starts_with("--target"))
161 10 explicitlySelectedTarget = true;
162
1/2
✓ Branch 83 → 84 taken 4042 times.
✗ Branch 83 → 449 not taken.
 4042 argv.push_back(arg.c_str());
163 }
164
1/2
✓ Branch 94 → 95 taken 1315 times.
✗ Branch 94 → 733 not taken.
 1315 Driver driver(cliOptions, true);
165
1/2
✓ Branch 97 → 98 taken 1315 times.
✗ Branch 97 → 731 not taken.
 1315 driver.parse(static_cast<int>(argv.size()), argv.data());
166
1/2
✓ Branch 98 → 99 taken 1315 times.
✗ Branch 98 → 731 not taken.
 1315 driver.enrich();
167
168 // If this is a cross-compilation test, we want to emit the target information in IR. For this we need to set native to false
169
2/2
✓ Branch 99 → 100 taken 10 times.
✓ Branch 99 → 101 taken 1305 times.
 1315 if (explicitlySelectedTarget)
170 10 cliOptions.isNativeTarget = false;
171
172 // Redirect all build artifacts to a per-test directory. Driver::enrich() resets these to shared paths (e.g. "./" and a
173 // shared temp cache dir), which would cause concurrently running test processes to clobber each other's object files and
174 // executables. Using a unique directory per test keeps parallel runs (e.g. via gtest-parallel) collision-free.
175
1/2
✓ Branch 101 → 102 taken 1315 times.
✗ Branch 101 → 731 not taken.
 1315 const std::filesystem::path artifactDir = TestUtil::prepareArtifactDir(testCase);
176
1/2
✓ Branch 102 → 103 taken 1315 times.
✗ Branch 102 → 729 not taken.
 1315 const std::filesystem::path executablePath = TestUtil::getExecutablePath(artifactDir);
177
1/2
✓ Branch 103 → 104 taken 1315 times.
✗ Branch 103 → 727 not taken.
 1315 cliOptions.outputDir = artifactDir;
178
2/4
✓ Branch 104 → 105 taken 1315 times.
✗ Branch 104 → 453 not taken.
✓ Branch 105 → 106 taken 1315 times.
✗ Branch 105 → 451 not taken.
 1315 cliOptions.cacheDir = artifactDir / "cache";
179
1/2
✓ Branch 109 → 110 taken 1315 times.
✗ Branch 109 → 727 not taken.
 1315 std::filesystem::create_directories(cliOptions.cacheDir);
180
181 // Instantiate GlobalResourceManager
182
1/2
✓ Branch 110 → 111 taken 1315 times.
✗ Branch 110 → 727 not taken.
 1315 GlobalResourceManager resourceManager(cliOptions);
183
184 try {
185 // Create source file instance for main source file
186
2/4
✓ Branch 113 → 114 taken 1315 times.
✗ Branch 113 → 457 not taken.
✓ Branch 114 → 115 taken 1315 times.
✗ Branch 114 → 455 not taken.
 1315 SourceFile *mainSourceFile = resourceManager.createSourceFile(nullptr, MAIN_FILE_NAME, cliOptions.mainSourceFile, false);
187
188 // Run Lexer and Parser
189
2/2
✓ Branch 117 → 118 taken 1311 times.
✓ Branch 117 → 686 taken 4 times.
 1315 mainSourceFile->runLexer();
190
1/2
✓ Branch 118 → 119 taken 1311 times.
✗ Branch 118 → 686 not taken.
 1311 mainSourceFile->runParser();
191
192 // Build and optimize AST
193
2/2
✓ Branch 119 → 120 taken 1295 times.
✓ Branch 119 → 686 taken 16 times.
 1311 mainSourceFile->runASTBuilder();
194
195 // Check AST
196
3/6
✓ Branch 122 → 123 taken 1295 times.
✗ Branch 122 → 465 not taken.
✓ Branch 123 → 124 taken 1295 times.
✗ Branch 123 → 463 not taken.
✓ Branch 124 → 125 taken 1295 times.
✗ Branch 124 → 461 not taken.
 1295 TestUtil::checkRefMatch(testCase.testPath / REF_NAME_SYNTAX_TREE, [&] {
197 14 mainSourceFile->runASTVisualizer();
198 14 return mainSourceFile->compilerOutput.astString;
199 });
200
201 // Execute import collector and semantic analysis stages
202
2/2
✓ Branch 129 → 130 taken 1283 times.
✓ Branch 129 → 686 taken 12 times.
 1295 mainSourceFile->runImportCollector();
203
2/2
✓ Branch 130 → 131 taken 1241 times.
✓ Branch 130 → 686 taken 42 times.
 1283 mainSourceFile->runSymbolTableBuilder();
204
2/2
✓ Branch 131 → 132 taken 859 times.
✓ Branch 131 → 686 taken 382 times.
 1241 mainSourceFile->runMiddleEnd(); // TypeChecker pre + post
205
206 // Check symbol table output (check happens here to include updates from type checker)
207
3/6
✓ Branch 134 → 135 taken 859 times.
✗ Branch 134 → 479 not taken.
✓ Branch 135 → 136 taken 859 times.
✗ Branch 135 → 477 not taken.
✓ Branch 136 → 137 taken 859 times.
✗ Branch 136 → 475 not taken.
 1718 TestUtil::checkRefMatch(testCase.testPath / REF_NAME_SYMBOL_TABLE,
208
2/4
✓ Branch 3 → 4 taken 16 times.
✗ Branch 3 → 11 not taken.
✓ Branch 4 → 5 taken 16 times.
✗ Branch 4 → 9 not taken.
 1750 [&] { return mainSourceFile->globalScope->getSymbolTableJSON().dump(/*indent=*/2); });
209
210 // Fail if an error was expected
211 − if (TestUtil::doesRefExist(testCase.testPath / REF_NAME_ERROR_OUTPUT)) // GCOV_EXCL_LINE
212 − FAIL() << "Expected error, but got no error"; // GCOV_EXCL_LINE
213
214 // Check dependency graph
215
3/6
✓ Branch 156 → 157 taken 859 times.
✗ Branch 156 → 505 not taken.
✓ Branch 157 → 158 taken 859 times.
✗ Branch 157 → 503 not taken.
✓ Branch 158 → 159 taken 859 times.
✗ Branch 158 → 501 not taken.
 859 TestUtil::checkRefMatch(testCase.testPath / REF_NAME_DEP_GRAPH, [&] {
216 4 mainSourceFile->runDependencyGraphVisualizer();
217 4 return mainSourceFile->compilerOutput.depGraphString;
218 });
219
220 // Run backend for all dependencies
221
5/8
✓ Branch 163 → 164 taken 859 times.
✗ Branch 163 → 515 not taken.
✓ Branch 164 → 165 taken 859 times.
✗ Branch 164 → 515 not taken.
✓ Branch 165 → 166 taken 859 times.
✗ Branch 165 → 515 not taken.
✓ Branch 171 → 167 taken 923 times.
✓ Branch 171 → 172 taken 859 times.
 1782 for (SourceFile *sourceFile : mainSourceFile->dependencies | std::views::values)
222
1/2
✓ Branch 168 → 169 taken 923 times.
✗ Branch 168 → 515 not taken.
 923 sourceFile->runBackEnd();
223
224 // Keep track of the opt level the IR in the module was generated with
225 859 OptLevel generatedOptLevel = cliOptions.optLevel;
226 // Execute IR generator in normal or debug mode
227
1/2
✓ Branch 172 → 173 taken 859 times.
✗ Branch 172 → 686 not taken.
 859 mainSourceFile->runIRGenerator();
228
229 // Coverage counters are woven in by the optimizer pipeline, which the IR-check loop below only runs for opt levels
230 // that have a reference file to compare against. Most test cases only check execution output and have no IR
231 // reference at all, so without this, their linked binary would never actually carry the GCOV counters. Force one
232 // optimizer run at the default opt level in that case.
233
2/2
✓ Branch 173 → 174 taken 421 times.
✓ Branch 173 → 177 taken 438 times.
 859 if (cliOptions.instrumentation.codeCoverage) {
234
1/2
✓ Branch 174 → 175 taken 421 times.
✗ Branch 174 → 686 not taken.
 421 const bool anyOptIrRefExists = std::ranges::any_of(
235
3/6
✓ Branch 2 → 3 taken 1663 times.
✗ Branch 2 → 14 not taken.
✓ Branch 3 → 4 taken 1663 times.
✗ Branch 3 → 12 not taken.
✓ Branch 4 → 5 taken 1663 times.
✗ Branch 4 → 10 not taken.
 1663 REF_NAME_OPT_IR, [&](const char *const ref) { return TestUtil::doesRefExist(testCase.testPath / ref); });
236
2/2
✓ Branch 175 → 176 taken 247 times.
✓ Branch 175 → 177 taken 174 times.
 421 if (!anyOptIrRefExists)
237
1/2
✓ Branch 176 → 177 taken 247 times.
✗ Branch 176 → 686 not taken.
 247 mainSourceFile->runDefaultIROptimizer();
238 }
239
240 // Check IR code
241
2/2
✓ Branch 188 → 178 taken 5154 times.
✓ Branch 188 → 189 taken 859 times.
 6013 for (uint8_t i = 0; i <= 5; i++) {
242
1/2
✓ Branch 182 → 183 taken 5154 times.
✗ Branch 182 → 516 not taken.
 5154 TestUtil::checkRefMatch(
243
2/4
✓ Branch 180 → 181 taken 5154 times.
✗ Branch 180 → 520 not taken.
✓ Branch 181 → 182 taken 5154 times.
✗ Branch 181 → 518 not taken.
 10308 testCase.testPath / REF_NAME_OPT_IR[i],
244
1/2
✓ Branch 179 → 180 taken 5154 times.
✗ Branch 179 → 524 not taken.
 10308 [&] {
245 429 cliOptions.optLevel = static_cast<OptLevel>(i);
246
247 // The IR generator emits parts of the IR depending on the selected opt level, so the IR has to be
248 // generated from scratch whenever we check the dump of an opt level other than the one the current module
249 // was built for. Otherwise all dumps would show the IR for the opt level that was active when the IR
250 // generator ran for the first time.
251
2/2
✓ Branch 2 → 3 taken 79 times.
✓ Branch 2 → 13 taken 350 times.
 429 if (cliOptions.optLevel != generatedOptLevel) {
252
2/2
✓ Branch 3 → 4 taken 1 time.
✓ Branch 3 → 11 taken 78 times.
 79 if (cliOptions.useLTO) {
253 // The bitcode linker moves the module of every source file into the LTO module, so all of them have to
254 // be re-generated. The LTO module itself is started from scratch to not link the same symbols twice.
255
1/2
✓ Branch 4 → 5 taken 1 time.
✗ Branch 4 → 20 not taken.
 1 resourceManager.ltoModule = std::make_unique<llvm::Module>(LTO_FILE_NAME, resourceManager.ltoContext);
256 1 std::unordered_set<const SourceFile *> visitedSet;
257
1/2
✓ Branch 8 → 9 taken 1 time.
✗ Branch 8 → 21 not taken.
 1 reGenerateIRForLTO(mainSourceFile, visitedSet);
258 1 } else {
259 78 reGenerateIR(mainSourceFile);
260 }
261 79 generatedOptLevel = cliOptions.optLevel;
262 }
263
264
2/2
✓ Branch 13 → 14 taken 1 time.
✓ Branch 13 → 17 taken 428 times.
 429 if (cliOptions.useLTO) {
265 1 mainSourceFile->runPreLinkIROptimizer();
266 1 mainSourceFile->runBitcodeLinker();
267 1 mainSourceFile->runPostLinkIROptimizer();
268 } else {
269 428 mainSourceFile->runDefaultIROptimizer();
270 }
271
272 429 return mainSourceFile->compilerOutput.irOptString;
273 },
274 10308 [&](std::string &expectedOutput, std::string &actualOutput) {
275
2/2
✓ Branch 4 → 5 taken 7 times.
✓ Branch 4 → 7 taken 212 times.
 438 if (cliOptions.instrumentation.emitsDebugInfo()) {
276 // Remove the lines, containing paths on the local file system
277 7 TestUtil::eraseLinesBySubstring(expectedOutput, " = !DIFile(filename:");
278 7 TestUtil::eraseLinesBySubstring(actualOutput, " = !DIFile(filename:");
279 }
280 219 },
281 true);
282 }
283
284 // Link the bitcode if not happened yet
285
3/4
✓ Branch 189 → 190 taken 1 time.
✓ Branch 189 → 192 taken 858 times.
✗ Branch 190 → 191 not taken.
✓ Branch 190 → 192 taken 1 time.
 859 if (cliOptions.useLTO && cliOptions.optLevel == OptLevel::O0)
286 ✗ mainSourceFile->runBitcodeLinker();
287
288 // Check assembly code (only when not running test on GitHub Actions)
289 859 bool objectFilesEmitted = false;
290 // GCOV_EXCL_START
291 − if (!testDriverCliOptions.isGitHubActions) {
292 − TestUtil::checkRefMatch(testCase.testPath / REF_NAME_ASM, [&] {
293 − mainSourceFile->runObjectEmitter();
294 − objectFilesEmitted = true;
295
296 − return mainSourceFile->compilerOutput.asmString;
297 });
298 }
299 // GCOV_EXCL_STOP
300
301 // Check warnings
302
1/2
✓ Branch 203 → 204 taken 859 times.
✗ Branch 203 → 686 not taken.
 859 mainSourceFile->collectAndPrintWarnings();
303
3/6
✓ Branch 206 → 207 taken 859 times.
✗ Branch 206 → 549 not taken.
✓ Branch 207 → 208 taken 859 times.
✗ Branch 207 → 547 not taken.
✓ Branch 208 → 209 taken 859 times.
✗ Branch 208 → 545 not taken.
 859 TestUtil::checkRefMatch(testCase.testPath / REF_NAME_WARNING_OUTPUT, [&] {
304
1/2
✓ Branch 2 → 3 taken 50 times.
✗ Branch 2 → 27 not taken.
 50 std::stringstream actualWarningString;
305
2/2
✓ Branch 18 → 5 taken 52 times.
✓ Branch 18 → 19 taken 50 times.
 152 for (const CompilerWarning &warning : mainSourceFile->compilerOutput.warnings)
306
2/4
✓ Branch 7 → 8 taken 52 times.
✗ Branch 7 → 24 not taken.
✓ Branch 8 → 9 taken 52 times.
✗ Branch 8 → 24 not taken.
 52 actualWarningString << warning.warningMessage << "\n";
307
1/2
✓ Branch 19 → 20 taken 50 times.
✗ Branch 19 → 25 not taken.
 100 return actualWarningString.str();
308 50 });
309
310 // Do linking and conclude compilation
311
3/6
✓ Branch 213 → 214 taken 859 times.
✗ Branch 213 → 563 not taken.
✓ Branch 214 → 215 taken 859 times.
✗ Branch 214 → 561 not taken.
✓ Branch 215 → 216 taken 859 times.
✗ Branch 215 → 559 not taken.
 859 const bool needsNormalRunForOutput = TestUtil::doesRefExist(testCase.testPath / REF_NAME_EXECUTION_OUTPUT);
312
3/6
✓ Branch 218 → 219 taken 859 times.
✗ Branch 218 → 569 not taken.
✓ Branch 219 → 220 taken 859 times.
✗ Branch 219 → 567 not taken.
✓ Branch 220 → 221 taken 859 times.
✗ Branch 220 → 565 not taken.
 859 const bool needsNormalRunForExitCode = TestUtil::doesRefExist(testCase.testPath / REF_NAME_EXIT_CODE);
313
3/6
✓ Branch 223 → 224 taken 859 times.
✗ Branch 223 → 575 not taken.
✓ Branch 224 → 225 taken 859 times.
✗ Branch 224 → 573 not taken.
✓ Branch 225 → 226 taken 859 times.
✗ Branch 225 → 571 not taken.
 859 const bool needsDebuggerRun = TestUtil::doesRefExist(testCase.testPath / REF_NAME_GDB_OUTPUT);
314
6/6
✓ Branch 228 → 229 taken 133 times.
✓ Branch 228 → 231 taken 726 times.
✓ Branch 229 → 230 taken 110 times.
✓ Branch 229 → 231 taken 23 times.
✓ Branch 230 → 231 taken 2 times.
✓ Branch 230 → 238 taken 108 times.
 859 if (needsNormalRunForOutput || needsNormalRunForExitCode || needsDebuggerRun) {
315 // Emit main source file object if not done already
316
1/2
✓ Branch 231 → 232 taken 751 times.
✗ Branch 231 → 233 not taken.
 751 if (!objectFilesEmitted)
317
1/2
✓ Branch 232 → 233 taken 751 times.
✗ Branch 232 → 686 not taken.
 751 mainSourceFile->runObjectEmitter();
318
319 // Conclude the compilation
320
1/2
✓ Branch 233 → 234 taken 751 times.
✗ Branch 233 → 686 not taken.
 751 mainSourceFile->concludeCompilation();
321
322 // Prepare linker
323
1/2
✓ Branch 234 → 235 taken 751 times.
✗ Branch 234 → 686 not taken.
 751 resourceManager.linker.outputPath = executablePath;
324
325 // Prepare and run linker
326
1/2
✓ Branch 235 → 236 taken 751 times.
✗ Branch 235 → 686 not taken.
 751 resourceManager.linker.prepare();
327
1/2
✓ Branch 236 → 237 taken 751 times.
✗ Branch 236 → 686 not taken.
 751 resourceManager.linker.run();
328
1/2
✓ Branch 237 → 238 taken 751 times.
✗ Branch 237 → 686 not taken.
 751 resourceManager.linker.cleanup();
329 }
330
331 // Check type registry output
332
3/6
✓ Branch 240 → 241 taken 859 times.
✗ Branch 240 → 581 not taken.
✓ Branch 241 → 242 taken 859 times.
✗ Branch 241 → 579 not taken.
✓ Branch 242 → 243 taken 859 times.
✗ Branch 242 → 577 not taken.
 871 TestUtil::checkRefMatch(testCase.testPath / REF_NAME_TYPE_REGISTRY, [&] { return TypeRegistry::dump(); });
333
334 // Check cache stats output
335
3/6
✓ Branch 249 → 250 taken 859 times.
✗ Branch 249 → 595 not taken.
✓ Branch 250 → 251 taken 859 times.
✗ Branch 250 → 593 not taken.
✓ Branch 251 → 252 taken 859 times.
✗ Branch 251 → 591 not taken.
 859 TestUtil::checkRefMatch(testCase.testPath / REF_NAME_CACHE_STATS, [&] {
336
1/2
✓ Branch 2 → 3 taken 2 times.
✗ Branch 2 → 31 not taken.
 2 std::stringstream cacheStats;
337
3/6
✓ Branch 3 → 4 taken 2 times.
✗ Branch 3 → 22 not taken.
✓ Branch 4 → 5 taken 2 times.
✗ Branch 4 → 20 not taken.
✓ Branch 5 → 6 taken 2 times.
✗ Branch 5 → 20 not taken.
 2 cacheStats << FunctionManager::dumpLookupCacheStatistics() << std::endl;
338
3/6
✓ Branch 7 → 8 taken 2 times.
✗ Branch 7 → 25 not taken.
✓ Branch 8 → 9 taken 2 times.
✗ Branch 8 → 23 not taken.
✓ Branch 9 → 10 taken 2 times.
✗ Branch 9 → 23 not taken.
 2 cacheStats << StructManager::dumpLookupCacheStatistics() << std::endl;
339
3/6
✓ Branch 11 → 12 taken 2 times.
✗ Branch 11 → 28 not taken.
✓ Branch 12 → 13 taken 2 times.
✗ Branch 12 → 26 not taken.
✓ Branch 13 → 14 taken 2 times.
✗ Branch 13 → 26 not taken.
 2 cacheStats << InterfaceManager::dumpLookupCacheStatistics() << std::endl;
340
1/2
✓ Branch 15 → 16 taken 2 times.
✗ Branch 15 → 29 not taken.
 4 return cacheStats.str();
341 2 });
342
343
3/6
✓ Branch 256 → 257 taken 859 times.
✗ Branch 256 → 609 not taken.
✓ Branch 257 → 258 taken 859 times.
✗ Branch 257 → 607 not taken.
✓ Branch 258 → 259 taken 859 times.
✗ Branch 258 → 605 not taken.
 859 const bool checkExecutionOutput = TestUtil::doesRefExist(testCase.testPath / REF_NAME_EXECUTION_OUTPUT);
344
3/6
✓ Branch 261 → 262 taken 859 times.
✗ Branch 261 → 615 not taken.
✓ Branch 262 → 263 taken 859 times.
✗ Branch 262 → 613 not taken.
✓ Branch 263 → 264 taken 859 times.
✗ Branch 263 → 611 not taken.
 859 const bool checkExecutionExitCode = TestUtil::doesRefExist(testCase.testPath / REF_NAME_EXIT_CODE);
345
4/4
✓ Branch 266 → 267 taken 133 times.
✓ Branch 266 → 268 taken 726 times.
✓ Branch 267 → 268 taken 23 times.
✓ Branch 267 → 324 taken 110 times.
 859 if (checkExecutionOutput || checkExecutionExitCode) {
346
2/4
✓ Branch 268 → 269 taken 749 times.
✗ Branch 268 → 619 not taken.
✓ Branch 269 → 270 taken 749 times.
✗ Branch 269 → 617 not taken.
 749 const std::filesystem::path cliFlagsFile = testCase.testPath / INPUT_NAME_CLI_FLAGS;
347 // Execute binary
348
1/2
✓ Branch 271 → 272 taken 749 times.
✗ Branch 271 → 669 not taken.
 749 std::stringstream cmd;
349
1/2
✗ Branch 272 → 273 not taken.
✓ Branch 272 → 274 taken 749 times.
 749 if (testDriverCliOptions.enableLeakDetection)
350 ✗ cmd << "valgrind -q --leak-check=full --suppressions=../../valgrind.supp --num-callers=100 --error-exitcode=1 ";
351
2/4
✓ Branch 274 → 275 taken 749 times.
✗ Branch 274 → 622 not taken.
✓ Branch 275 → 276 taken 749 times.
✗ Branch 275 → 620 not taken.
 749 cmd << executablePath.string();
352
3/4
✓ Branch 277 → 278 taken 749 times.
✗ Branch 277 → 667 not taken.
✓ Branch 278 → 279 taken 6 times.
✓ Branch 278 → 285 taken 743 times.
 749 if (exists(cliFlagsFile))
353
4/8
✓ Branch 279 → 280 taken 6 times.
✗ Branch 279 → 667 not taken.
✓ Branch 280 → 281 taken 6 times.
✗ Branch 280 → 625 not taken.
✓ Branch 281 → 282 taken 6 times.
✗ Branch 281 → 623 not taken.
✓ Branch 282 → 283 taken 6 times.
✗ Branch 282 → 623 not taken.
 6 cmd << " " << TestUtil::getFileContentLinesVector(cliFlagsFile).at(0);
354
2/4
✓ Branch 285 → 286 taken 749 times.
✗ Branch 285 → 628 not taken.
✓ Branch 286 → 287 taken 749 times.
✗ Branch 286 → 626 not taken.
 749 const auto [output, exitCode] = SystemUtil::exec(cmd.str(), checkExecutionOutput);
355
356 // Check if the execution output matches the expected output
357 726 const auto getActualOutput = [&] { return output; };
358
3/6
✓ Branch 290 → 291 taken 749 times.
✗ Branch 290 → 633 not taken.
✓ Branch 291 → 292 taken 749 times.
✗ Branch 291 → 631 not taken.
✓ Branch 292 → 293 taken 749 times.
✗ Branch 292 → 629 not taken.
 749 TestUtil::checkRefMatch(testCase.testPath / REF_NAME_EXECUTION_OUTPUT, getActualOutput);
359
360 #if not OS_WINDOWS // Windows does not give us the exit code, so we cannot check it on Windows
361 // Check if the exit code matches the expected one
362 // If no exit code ref file exists, check against 0
363 51 const auto getActualExitCode = [&] { return std::to_string(exitCode); };
364
3/6
✓ Branch 299 → 300 taken 749 times.
✗ Branch 299 → 646 not taken.
✓ Branch 300 → 301 taken 749 times.
✗ Branch 300 → 644 not taken.
✓ Branch 301 → 302 taken 749 times.
✗ Branch 301 → 642 not taken.
 749 const bool refExists = TestUtil::checkRefMatch(testCase.testPath / REF_NAME_EXIT_CODE, getActualExitCode);
365
2/2
✓ Branch 306 → 307 taken 698 times.
✓ Branch 306 → 320 taken 51 times.
 749 if (!refExists) {
366
2/12
✓ Branch 307 → 308 taken 698 times.
✗ Branch 307 → 655 not taken.
✗ Branch 309 → 310 not taken.
✓ Branch 309 → 318 taken 698 times.
✗ Branch 310 → 311 not taken.
✗ Branch 310 → 661 not taken.
✗ Branch 311 → 312 not taken.
✗ Branch 311 → 659 not taken.
✗ Branch 313 → 314 not taken.
✗ Branch 313 → 658 not taken.
✗ Branch 314 → 315 not taken.
✗ Branch 314 → 656 not taken.
 698 EXPECT_EQ(0, exitCode) << "Program exited with non-zero exit code";
367 }
368 #endif
369 749 }
370
371 // Check if the debugger output matches the expected output
372 // GCOV_EXCL_START
373 − if (!testDriverCliOptions.isGitHubActions) { // GDB tests are currently not support on GH actions
374 − TestUtil::checkRefMatch(
375 − testCase.testPath / REF_NAME_GDB_OUTPUT,
376 − [&] {
377 // Execute debugger script
378 − std::filesystem::path gdbScriptPath = testCase.testPath / CTL_DEBUG_SCRIPT;
379 − EXPECT_TRUE(std::filesystem::exists(gdbScriptPath)) << "Debug output requested, but debug script not found";
380 − gdbScriptPath.make_preferred();
381 // -nx: ignore any local/system .gdbinit so a developer's own GDB setup (e.g. globally installed
382 // pretty printers) cannot change test output; tests that want the pretty printers source them
383 // explicitly from within their debug script instead.
384 − const std::string cmd = "gdb -nx -x " + gdbScriptPath.string() + " " + executablePath.string();
385 − const auto [output, exitCode] = SystemUtil::exec(cmd);
386
387 #if not OS_WINDOWS // Windows does not give us the exit code, so we cannot check it on Windows
388 − EXPECT_EQ(0, exitCode) << "GDB exited with non-zero exit code when running debug script";
389 #endif
390
391 − return output;
392 − },
393 − [&](std::string &expectedOutput, std::string &actualOutput) {
394 // Do not compare against the GDB header
395 − TestUtil::eraseGDBHeader(expectedOutput);
396 − TestUtil::eraseGDBHeader(actualOutput);
397 − });
398 }
399 // GCOV_EXCL_STOP
400
4/7
✗ Branch 687 → 688 not taken.
✓ Branch 687 → 689 taken 2 times.
✓ Branch 687 → 692 taken 16 times.
✓ Branch 687 → 695 taken 160 times.
✓ Branch 687 → 698 taken 278 times.
✗ Branch 687 → 701 not taken.
✗ Branch 687 → 704 not taken.
 456 } catch (LexerError &error) {
401
1/2
✓ Branch 690 → 691 taken 2 times.
✗ Branch 690 → 707 not taken.
 2 TestUtil::handleError(testCase, error);
402
1/2
✓ Branch 691 → 336 taken 2 times.
✗ Branch 691 → 725 not taken.
 18 } catch (ParserError &error) {
403
1/2
✓ Branch 693 → 694 taken 16 times.
✗ Branch 693 → 709 not taken.
 16 TestUtil::handleError(testCase, error);
404
1/2
✓ Branch 694 → 336 taken 16 times.
✗ Branch 694 → 725 not taken.
 176 } catch (SemanticError &error) {
405
1/2
✓ Branch 696 → 697 taken 160 times.
✗ Branch 696 → 711 not taken.
 160 TestUtil::handleError(testCase, error);
406
1/2
✓ Branch 697 → 336 taken 160 times.
✗ Branch 697 → 725 not taken.
 438 } catch (CompilerError &error) {
407
1/2
✓ Branch 699 → 700 taken 278 times.
✗ Branch 699 → 713 not taken.
 278 TestUtil::handleError(testCase, error);
408 278 } catch (LinkerError &error) {
409 ✗ TestUtil::handleError(testCase, error);
410 − } catch (std::exception &error) { // GCOV_EXCL_LINE
411 − TestUtil::handleError(testCase, error); // GCOV_EXCL_LINE
412 − } // GCOV_EXCL_LINE
413
414
3/6
✓ Branch 336 → 337 taken 1315 times.
✗ Branch 336 → 724 not taken.
✓ Branch 337 → 338 taken 1315 times.
✗ Branch 337 → 721 not taken.
✓ Branch 338 → 339 taken 1315 times.
✗ Branch 338 → 719 not taken.
 1315 SUCCEED();
415
8/16
✓ Branch 343 → 344 taken 1315 times.
✗ Branch 343 → 345 not taken.
✓ Branch 348 → 349 taken 1315 times.
✗ Branch 348 → 350 not taken.
✓ Branch 353 → 354 taken 1315 times.
✗ Branch 353 → 355 not taken.
✓ Branch 358 → 359 taken 1315 times.
✗ Branch 358 → 360 not taken.
✓ Branch 363 → 364 taken 1315 times.
✗ Branch 363 → 365 not taken.
✓ Branch 368 → 369 taken 1315 times.
✗ Branch 368 → 370 not taken.
✓ Branch 373 → 374 taken 1315 times.
✗ Branch 373 → 375 not taken.
✓ Branch 378 → 379 taken 1315 times.
✗ Branch 378 → 381 not taken.
 10520 }
416
417 /**
418 * Runs a test case against the bootstrap compiler, built at the start of the test run. The bootstrap compiler is invoked
419 * like the host compiler would be invoked by a user, since it cannot be driven stage by stage from here.
420 *
421 * The bootstrap compiler is still incomplete, so only the reference outputs it can already produce are checked: the
422 * serialized AST, the raised error, the IR code, and the execution output and exit code of the compiled program. A missing
423 * error only fails the test for the error kinds listed in BOOTSTRAP_SUPPORTED_ERROR_PREFIXES. Beyond that, each test case
424 * checks that the bootstrap compiler runs through all of its implemented stages without crashing or raising an unexpected
425 * error.
426 */
427 ✗ static void execBootstrapTestCase(const TestCase &testCase) {
428 // Check if test is disabled
429 ✗ if (TestUtil::isDisabled(testCase))
430 ✗ GTEST_SKIP();
431
432 ✗ const std::filesystem::path mainSourceFilePath = testCase.testPath / REF_NAME_SOURCE;
433 ✗ const std::filesystem::path artifactDir = TestUtil::prepareArtifactDir(testCase);
434 ✗ const std::filesystem::path executablePath = TestUtil::getExecutablePath(artifactDir);
435 ✗ const bool checkAST = TestUtil::doesRefExist(testCase.testPath / REF_NAME_SYNTAX_TREE);
436 ✗ const bool checkExecutionOutput = TestUtil::doesRefExist(testCase.testPath / REF_NAME_EXECUTION_OUTPUT);
437 ✗ const bool checkExecutionExitCode = TestUtil::doesRefExist(testCase.testPath / REF_NAME_EXIT_CODE);
438 ✗ const bool needsExecutable = checkExecutionOutput || checkExecutionExitCode;
439
440 // Assemble the command line, mirroring the one the test runner passes to the host compiler
441 ✗ const auto buildArgs = [&](const std::filesystem::path &outputPath) {
442 ✗ std::vector<std::string> args = {"build", "--test-mode"};
443 ✗ TestUtil::parseTestArgs(mainSourceFilePath, args);
444 ✗ if (exists(testCase.testPath / CTL_RUN_BUILTIN_TESTS))
445 ✗ args.emplace_back("--no-entry");
446 ✗ args.emplace_back("--output");
447 ✗ args.push_back(outputPath.string());
448 ✗ return args;
449 ✗ };
450 ✗ std::vector<std::string> args = buildArgs(executablePath);
451 // Like the host test runner, only link an executable if it gets executed afterwards
452 ✗ if (!needsExecutable) {
453 ✗ args.emplace_back("--output-container");
454 ✗ args.emplace_back("obj");
455 }
456 ✗ if (checkAST)
457 ✗ args.emplace_back("--dump-ast");
458 ✗ args.push_back(mainSourceFilePath.string());
459
460 // Run the bootstrap compiler
461 ✗ const auto [output, exitCode] = SystemUtil::exec(testDriverCliOptions.bootstrapCompilerPath, args, true);
462 − if (testDriverCliOptions.isVerbose) // GCOV_EXCL_LINE
463 − std::cout << "Bootstrap compiler output:\n" << output; // GCOV_EXCL_LINE
464
465 // Check if the bootstrap compiler raised an error
466 ✗ const std::filesystem::path errorRefPath = testCase.testPath / REF_NAME_ERROR_OUTPUT;
467 ✗ if (const std::optional<std::string> errorMessage = BootstrapUtil::extractErrorMessage(output)) {
468 ✗ if (!TestUtil::doesRefExist(errorRefPath))
469 ✗ FAIL() << "Expected no error, but got: " << *errorMessage;
470 ✗ TestUtil::checkRefMatch(errorRefPath, [&] { return *errorMessage; });
471 ✗ return;
472 ✗ }
473 ✗ if (exitCode != 0)
474 ✗ FAIL() << "Bootstrap compiler exited with code " << exitCode << ":\n" << output;
475
476 // Check AST
477 ✗ TestUtil::checkRefMatch(testCase.testPath / REF_NAME_SYNTAX_TREE, [&] {
478 ✗ const std::optional<std::string> astString = BootstrapUtil::extractSerializedAST(output);
479 ✗ EXPECT_TRUE(astString.has_value()) << "Bootstrap compiler did not dump the AST:\n" << output;
480 ✗ return astString.value_or("");
481 ✗ });
482
483 // Fail if an error was expected, that the bootstrap compiler is already able to raise
484 ✗ if (TestUtil::doesRefExist(errorRefPath) && BootstrapUtil::isErrorSupported(errorRefPath))
485 ✗ FAIL() << "Expected error, but got no error";
486
487 // Check IR code. The host checks the IR after running the optimizer pipeline of each opt level, for which a reference exists.
488 // The bootstrap compiler dumps the optimized IR of every source file into the output dir, so it is compiled once per opt level.
489 ✗ for (uint8_t i = 0; i <= 5; i++) {
490 ✗ TestUtil::checkRefMatch(
491 ✗ testCase.testPath / REF_NAME_OPT_IR[i],
492 ✗ [&] {
493 ✗ const std::filesystem::path irArtifactDir = artifactDir / ("ir-O" + std::to_string(i));
494 ✗ std::filesystem::create_directories(irArtifactDir);
495 ✗ std::vector<std::string> irArgs = buildArgs(irArtifactDir / "object.o");
496 ✗ irArgs.emplace_back("-O" + std::string(1, BOOTSTRAP_OPT_LEVEL_NAMES[i]));
497 ✗ irArgs.emplace_back("--output-container");
498 ✗ irArgs.emplace_back("obj");
499 ✗ irArgs.emplace_back("--dump-ir");
500 ✗ irArgs.emplace_back("--dump-to-files");
501 ✗ irArgs.push_back(mainSourceFilePath.string());
502 ✗ const auto [irOutput, irExitCode] = SystemUtil::exec(testDriverCliOptions.bootstrapCompilerPath, irArgs, true);
503 ✗ EXPECT_EQ(0, irExitCode) << "Bootstrap compiler exited with code " << irExitCode << ":\n" << irOutput;
504 // With LTO, the bootstrap compiler dumps the IR of the LTO module after the post-link optimization
505 ✗ const bool useLTO = std::ranges::find(irArgs, "-lto") != irArgs.end();
506 ✗ const std::string irDumpName =
507 ✗ useLTO ? "source-ir-code-lto-post-link.ll" : "source-ir-code-O" + std::to_string(i) + ".ll";
508 ✗ const std::filesystem::path irDumpPath = irArtifactDir / irDumpName;
509 ✗ if (!exists(irDumpPath))
510 ✗ return std::string();
511 ✗ return FileUtil::getFileContent(irDumpPath);
512 ✗ },
513 ✗ [&](std::string &expectedOutput, std::string &actualOutput) {
514 // The LLVM C API, which the bootstrap compiler uses, cannot mark global values as dso_local
515 ✗ BootstrapUtil::eraseDSOLocalMarkers(expectedOutput);
516 ✗ BootstrapUtil::eraseDSOLocalMarkers(actualOutput);
517 ✗ },
518 true);
519 }
520
521 // Check execution output and exit code
522 ✗ if (needsExecutable) {
523 // Execute binary
524 ✗ std::stringstream cmd;
525 ✗ cmd << executablePath.string();
526 ✗ const std::filesystem::path cliFlagsFile = testCase.testPath / INPUT_NAME_CLI_FLAGS;
527 ✗ if (exists(cliFlagsFile))
528 ✗ cmd << " " << TestUtil::getFileContentLinesVector(cliFlagsFile).at(0);
529 ✗ const auto [programOutput, programExitCode] = SystemUtil::exec(cmd.str(), checkExecutionOutput);
530
531 // Check if the execution output matches the expected output
532 ✗ TestUtil::checkRefMatch(testCase.testPath / REF_NAME_EXECUTION_OUTPUT, [&] { return programOutput; });
533
534 #if not OS_WINDOWS // Windows does not give us the exit code, so we cannot check it on Windows
535 // Check if the exit code matches the expected one. If no exit code ref file exists, check against 0
536 ✗ const bool refExists = TestUtil::checkRefMatch(testCase.testPath / REF_NAME_EXIT_CODE,
537 ✗ [&] { return std::to_string(programExitCode); });
538 ✗ if (!refExists) {
539 ✗ EXPECT_EQ(0, programExitCode) << "Program exited with non-zero exit code";
540 }
541 #endif
542 ✗ }
543
544 ✗ SUCCEED();
545 ✗ }
546
547 /**
548 * Runs a test case against the host compiler or, in bootstrap mode, against the bootstrap compiler
549 */
550 1342 static void runTestCase(const TestCase &testCase) {
551
1/2
✗ Branch 2 → 3 not taken.
✓ Branch 2 → 4 taken 1342 times.
 1342 if (testDriverCliOptions.bootstrapMode)
552 ✗ execBootstrapTestCase(testCase);
553 else
554 1342 execTestCase(testCase);
555 1342 }
556
557 namespace {
558
559 /**
560 * Runs a lint test case against the bootstrap compiler: invokes its `lint` subcommand and compares the emitted
561 * findings against lint.out. The bootstrap compiler colorizes its findings, so the captured output is
562 * de-colorized before being matched against the plain-text reference.
563 */
564 ✗ void execBootstrapLinterTestCase(const TestCase &testCase) {
565 // Check if test is disabled
566 ✗ if (TestUtil::isDisabled(testCase))
567 ✗ GTEST_SKIP();
568
569 ✗ const std::filesystem::path mainSourceFilePath = testCase.testPath / REF_NAME_SOURCE;
570
571 // Assemble the command line
572 ✗ const std::vector<std::string> args = {"lint", mainSourceFilePath.string()};
573
574 // Run the bootstrap compiler
575 ✗ const auto [output, exitCode] = SystemUtil::exec(testDriverCliOptions.bootstrapCompilerPath, args, true);
576 − if (testDriverCliOptions.isVerbose) // GCOV_EXCL_LINE
577 − std::cout << "Bootstrap compiler output:\n" << output; // GCOV_EXCL_LINE
578
579 // Check if the bootstrap compiler raised an error
580 ✗ const std::filesystem::path errorRefPath = testCase.testPath / REF_NAME_ERROR_OUTPUT;
581 ✗ if (const std::optional<std::string> errorMessage = BootstrapUtil::extractErrorMessage(output)) {
582 ✗ if (!TestUtil::doesRefExist(errorRefPath))
583 ✗ FAIL() << "Expected no error, but got: " << *errorMessage;
584 ✗ TestUtil::checkRefMatch(errorRefPath, [&] { return *errorMessage; });
585 ✗ return;
586 ✗ }
587 ✗ if (exitCode != 0)
588 ✗ FAIL() << "Bootstrap compiler exited with code " << exitCode << ":\n" << output;
589
590 // Fail if an error was expected, that the bootstrap compiler is already able to raise
591 ✗ if (TestUtil::doesRefExist(errorRefPath) && BootstrapUtil::isErrorSupported(errorRefPath))
592 ✗ FAIL() << "Expected error, but got no error";
593
594 // Check lint findings against the reference (de-colorized)
595 ✗ TestUtil::checkRefMatch(testCase.testPath / REF_NAME_LINT_OUTPUT, [&] { return BootstrapUtil::stripAnsiCodes(output); });
596
597 ✗ SUCCEED();
598 ✗ }
599
600 } // namespace
601
602 class CommonTests : public ::testing::TestWithParam<TestCase> {};
603
7/16
✓ Branch 6 → 7 taken 2 times.
✗ Branch 6 → 53 not taken.
✓ Branch 10 → 11 taken 2 times.
✗ Branch 10 → 45 not taken.
✓ Branch 11 → 12 taken 2 times.
✗ Branch 11 → 43 not taken.
✓ Branch 14 → 15 taken 2 times.
✗ Branch 14 → 38 not taken.
✓ Branch 16 → 17 taken 2 times.
✗ Branch 16 → 34 not taken.
✓ Branch 18 → 19 taken 2 times.
✗ Branch 18 → 31 not taken.
✗ Branch 19 → 20 not taken.
✓ Branch 19 → 21 taken 2 times.
✗ Branch 31 → 32 not taken.
✗ Branch 31 → 33 not taken.
 58 TEST_P(CommonTests, ) { runTestCase(GetParam()); }
604
4/14
spice::testing::gtest_CommonTests_EvalGenerator_():
✓ Branch 2 → 3 taken 2 times.
✗ Branch 2 → 10 not taken.
✓ Branch 3 → 4 taken 2 times.
✗ Branch 3 → 8 not taken.
spice::testing::gtest_CommonTests_EvalGenerateName_(testing::TestParamInfo<spice::testing::TestCase> const&):
✗ Branch 3 → 4 not taken.
✓ Branch 3 → 12 taken 22 times.
✗ Branch 5 → 6 not taken.
✗ Branch 5 → 20 not taken.
✗ Branch 6 → 7 not taken.
✗ Branch 6 → 18 not taken.
✗ Branch 7 → 8 not taken.
✗ Branch 7 → 16 not taken.
✓ Branch 12 → 13 taken 22 times.
✗ Branch 12 → 24 not taken.
 48 INSTANTIATE_TEST_SUITE_P(, CommonTests, ::testing::ValuesIn(TestUtil::collectTestCases("common", false)),
605 TestUtil::NameResolver());
606
607 class LexerTests : public ::testing::TestWithParam<TestCase> {};
608
7/16
✓ Branch 6 → 7 taken 2 times.
✗ Branch 6 → 53 not taken.
✓ Branch 10 → 11 taken 2 times.
✗ Branch 10 → 45 not taken.
✓ Branch 11 → 12 taken 2 times.
✗ Branch 11 → 43 not taken.
✓ Branch 14 → 15 taken 2 times.
✗ Branch 14 → 38 not taken.
✓ Branch 16 → 17 taken 2 times.
✗ Branch 16 → 34 not taken.
✓ Branch 18 → 19 taken 2 times.
✗ Branch 18 → 31 not taken.
✗ Branch 19 → 20 not taken.
✓ Branch 19 → 21 taken 2 times.
✗ Branch 31 → 32 not taken.
✗ Branch 31 → 33 not taken.
 18 TEST_P(LexerTests, ) { runTestCase(GetParam()); }
609
4/14
spice::testing::gtest_LexerTests_EvalGenerator_():
✓ Branch 2 → 3 taken 2 times.
✗ Branch 2 → 10 not taken.
✓ Branch 3 → 4 taken 2 times.
✗ Branch 3 → 8 not taken.
spice::testing::gtest_LexerTests_EvalGenerateName_(testing::TestParamInfo<spice::testing::TestCase> const&):
✗ Branch 3 → 4 not taken.
✓ Branch 3 → 12 taken 2 times.
✗ Branch 5 → 6 not taken.
✗ Branch 5 → 20 not taken.
✗ Branch 6 → 7 not taken.
✗ Branch 6 → 18 not taken.
✗ Branch 7 → 8 not taken.
✗ Branch 7 → 16 not taken.
✓ Branch 12 → 13 taken 2 times.
✗ Branch 12 → 24 not taken.
 8 INSTANTIATE_TEST_SUITE_P(, LexerTests, ::testing::ValuesIn(TestUtil::collectTestCases("lexer", false)), TestUtil::NameResolver());
610
611 class ParserTests : public ::testing::TestWithParam<TestCase> {};
612
7/16
✓ Branch 6 → 7 taken 2 times.
✗ Branch 6 → 53 not taken.
✓ Branch 10 → 11 taken 2 times.
✗ Branch 10 → 45 not taken.
✓ Branch 11 → 12 taken 2 times.
✗ Branch 11 → 43 not taken.
✓ Branch 14 → 15 taken 2 times.
✗ Branch 14 → 38 not taken.
✓ Branch 16 → 17 taken 2 times.
✗ Branch 16 → 34 not taken.
✓ Branch 18 → 19 taken 2 times.
✗ Branch 18 → 31 not taken.
✗ Branch 19 → 20 not taken.
✓ Branch 19 → 21 taken 2 times.
✗ Branch 31 → 32 not taken.
✗ Branch 31 → 33 not taken.
 46 TEST_P(ParserTests, ) { runTestCase(GetParam()); }
613
4/14
spice::testing::gtest_ParserTests_EvalGenerator_():
✓ Branch 2 → 3 taken 2 times.
✗ Branch 2 → 10 not taken.
✓ Branch 3 → 4 taken 2 times.
✗ Branch 3 → 8 not taken.
spice::testing::gtest_ParserTests_EvalGenerateName_(testing::TestParamInfo<spice::testing::TestCase> const&):
✗ Branch 3 → 4 not taken.
✓ Branch 3 → 12 taken 16 times.
✗ Branch 5 → 6 not taken.
✗ Branch 5 → 20 not taken.
✗ Branch 6 → 7 not taken.
✗ Branch 6 → 18 not taken.
✗ Branch 7 → 8 not taken.
✗ Branch 7 → 16 not taken.
✓ Branch 12 → 13 taken 16 times.
✗ Branch 12 → 24 not taken.
 36 INSTANTIATE_TEST_SUITE_P(, ParserTests, ::testing::ValuesIn(TestUtil::collectTestCases("parser", false)),
614 TestUtil::NameResolver());
615
616 class SymbolTableBuilderTests : public ::testing::TestWithParam<TestCase> {};
617
7/16
✓ Branch 6 → 7 taken 2 times.
✗ Branch 6 → 53 not taken.
✓ Branch 10 → 11 taken 2 times.
✗ Branch 10 → 45 not taken.
✓ Branch 11 → 12 taken 2 times.
✗ Branch 11 → 43 not taken.
✓ Branch 14 → 15 taken 2 times.
✗ Branch 14 → 38 not taken.
✓ Branch 16 → 17 taken 2 times.
✗ Branch 16 → 34 not taken.
✓ Branch 18 → 19 taken 2 times.
✗ Branch 18 → 31 not taken.
✗ Branch 19 → 20 not taken.
✓ Branch 19 → 21 taken 2 times.
✗ Branch 31 → 32 not taken.
✗ Branch 31 → 33 not taken.
 114 TEST_P(SymbolTableBuilderTests, ) { runTestCase(GetParam()); }
618
4/14
spice::testing::gtest_SymbolTableBuilderTests_EvalGenerator_():
✓ Branch 2 → 3 taken 2 times.
✗ Branch 2 → 10 not taken.
✓ Branch 3 → 4 taken 2 times.
✗ Branch 3 → 8 not taken.
spice::testing::gtest_SymbolTableBuilderTests_EvalGenerateName_(testing::TestParamInfo<spice::testing::TestCase> const&):
✗ Branch 3 → 4 not taken.
✓ Branch 3 → 12 taken 50 times.
✗ Branch 5 → 6 not taken.
✗ Branch 5 → 20 not taken.
✗ Branch 6 → 7 not taken.
✗ Branch 6 → 18 not taken.
✗ Branch 7 → 8 not taken.
✗ Branch 7 → 16 not taken.
✓ Branch 12 → 13 taken 50 times.
✗ Branch 12 → 24 not taken.
 104 INSTANTIATE_TEST_SUITE_P(, SymbolTableBuilderTests, ::testing::ValuesIn(TestUtil::collectTestCases("symboltablebuilder", true)),
619 TestUtil::NameResolver());
620
621 class TypeCheckerTests : public ::testing::TestWithParam<TestCase> {};
622
7/16
✓ Branch 6 → 7 taken 2 times.
✗ Branch 6 → 53 not taken.
✓ Branch 10 → 11 taken 2 times.
✗ Branch 10 → 45 not taken.
✓ Branch 11 → 12 taken 2 times.
✗ Branch 11 → 43 not taken.
✓ Branch 14 → 15 taken 2 times.
✗ Branch 14 → 38 not taken.
✓ Branch 16 → 17 taken 2 times.
✗ Branch 16 → 34 not taken.
✓ Branch 18 → 19 taken 2 times.
✗ Branch 18 → 31 not taken.
✗ Branch 19 → 20 not taken.
✓ Branch 19 → 21 taken 2 times.
✗ Branch 31 → 32 not taken.
✗ Branch 31 → 33 not taken.
 986 TEST_P(TypeCheckerTests, ) { runTestCase(GetParam()); }
623
4/14
spice::testing::gtest_TypeCheckerTests_EvalGenerator_():
✓ Branch 2 → 3 taken 2 times.
✗ Branch 2 → 10 not taken.
✓ Branch 3 → 4 taken 2 times.
✗ Branch 3 → 8 not taken.
spice::testing::gtest_TypeCheckerTests_EvalGenerateName_(testing::TestParamInfo<spice::testing::TestCase> const&):
✗ Branch 3 → 4 not taken.
✓ Branch 3 → 12 taken 486 times.
✗ Branch 5 → 6 not taken.
✗ Branch 5 → 20 not taken.
✗ Branch 6 → 7 not taken.
✗ Branch 6 → 18 not taken.
✗ Branch 7 → 8 not taken.
✗ Branch 7 → 16 not taken.
✓ Branch 12 → 13 taken 486 times.
✗ Branch 12 → 24 not taken.
 976 INSTANTIATE_TEST_SUITE_P(, TypeCheckerTests, ::testing::ValuesIn(TestUtil::collectTestCases("typechecker", true)),
624 TestUtil::NameResolver());
625
626 class IRGeneratorTests : public ::testing::TestWithParam<TestCase> {};
627
7/16
✓ Branch 6 → 7 taken 2 times.
✗ Branch 6 → 53 not taken.
✓ Branch 10 → 11 taken 2 times.
✗ Branch 10 → 45 not taken.
✓ Branch 11 → 12 taken 2 times.
✗ Branch 11 → 43 not taken.
✓ Branch 14 → 15 taken 2 times.
✗ Branch 14 → 38 not taken.
✓ Branch 16 → 17 taken 2 times.
✗ Branch 16 → 34 not taken.
✓ Branch 18 → 19 taken 2 times.
✗ Branch 18 → 31 not taken.
✗ Branch 19 → 20 not taken.
✓ Branch 19 → 21 taken 2 times.
✗ Branch 31 → 32 not taken.
✗ Branch 31 → 33 not taken.
 894 TEST_P(IRGeneratorTests, ) { runTestCase(GetParam()); }
628
4/14
spice::testing::gtest_IRGeneratorTests_EvalGenerator_():
✓ Branch 2 → 3 taken 2 times.
✗ Branch 2 → 10 not taken.
✓ Branch 3 → 4 taken 2 times.
✗ Branch 3 → 8 not taken.
spice::testing::gtest_IRGeneratorTests_EvalGenerateName_(testing::TestParamInfo<spice::testing::TestCase> const&):
✗ Branch 3 → 4 not taken.
✓ Branch 3 → 12 taken 440 times.
✗ Branch 5 → 6 not taken.
✗ Branch 5 → 20 not taken.
✗ Branch 6 → 7 not taken.
✗ Branch 6 → 18 not taken.
✗ Branch 7 → 8 not taken.
✗ Branch 7 → 16 not taken.
✓ Branch 12 → 13 taken 440 times.
✗ Branch 12 → 24 not taken.
 884 INSTANTIATE_TEST_SUITE_P(, IRGeneratorTests, ::testing::ValuesIn(TestUtil::collectTestCases("irgenerator", true)),
629 TestUtil::NameResolver());
630
631 class StdTests : public ::testing::TestWithParam<TestCase> {};
632
7/16
✓ Branch 6 → 7 taken 2 times.
✗ Branch 6 → 53 not taken.
✓ Branch 10 → 11 taken 2 times.
✗ Branch 10 → 45 not taken.
✓ Branch 11 → 12 taken 2 times.
✗ Branch 11 → 43 not taken.
✓ Branch 14 → 15 taken 2 times.
✗ Branch 14 → 38 not taken.
✓ Branch 16 → 17 taken 2 times.
✗ Branch 16 → 34 not taken.
✓ Branch 18 → 19 taken 2 times.
✗ Branch 18 → 31 not taken.
✗ Branch 19 → 20 not taken.
✓ Branch 19 → 21 taken 2 times.
✗ Branch 31 → 32 not taken.
✗ Branch 31 → 33 not taken.
 618 TEST_P(StdTests, ) { runTestCase(GetParam()); }
633
4/14
spice::testing::gtest_StdTests_EvalGenerator_():
✓ Branch 2 → 3 taken 2 times.
✗ Branch 2 → 10 not taken.
✓ Branch 3 → 4 taken 2 times.
✗ Branch 3 → 8 not taken.
spice::testing::gtest_StdTests_EvalGenerateName_(testing::TestParamInfo<spice::testing::TestCase> const&):
✗ Branch 3 → 4 not taken.
✓ Branch 3 → 12 taken 302 times.
✗ Branch 5 → 6 not taken.
✗ Branch 5 → 20 not taken.
✗ Branch 6 → 7 not taken.
✗ Branch 6 → 18 not taken.
✗ Branch 7 → 8 not taken.
✗ Branch 7 → 16 not taken.
✓ Branch 12 → 13 taken 302 times.
✗ Branch 12 → 24 not taken.
 608 INSTANTIATE_TEST_SUITE_P(, StdTests, ::testing::ValuesIn(TestUtil::collectTestCases("std", true)), TestUtil::NameResolver());
634
635 class BenchmarkTests : public ::testing::TestWithParam<TestCase> {};
636
7/16
✓ Branch 6 → 7 taken 2 times.
✗ Branch 6 → 53 not taken.
✓ Branch 10 → 11 taken 2 times.
✗ Branch 10 → 45 not taken.
✓ Branch 11 → 12 taken 2 times.
✗ Branch 11 → 43 not taken.
✓ Branch 14 → 15 taken 2 times.
✗ Branch 14 → 38 not taken.
✓ Branch 16 → 17 taken 2 times.
✗ Branch 16 → 34 not taken.
✓ Branch 18 → 19 taken 2 times.
✗ Branch 18 → 31 not taken.
✗ Branch 19 → 20 not taken.
✓ Branch 19 → 21 taken 2 times.
✗ Branch 31 → 32 not taken.
✗ Branch 31 → 33 not taken.
 42 TEST_P(BenchmarkTests, ) { runTestCase(GetParam()); }
637
4/14
spice::testing::gtest_BenchmarkTests_EvalGenerator_():
✓ Branch 2 → 3 taken 2 times.
✗ Branch 2 → 10 not taken.
✓ Branch 3 → 4 taken 2 times.
✗ Branch 3 → 8 not taken.
spice::testing::gtest_BenchmarkTests_EvalGenerateName_(testing::TestParamInfo<spice::testing::TestCase> const&):
✗ Branch 3 → 4 not taken.
✓ Branch 3 → 12 taken 14 times.
✗ Branch 5 → 6 not taken.
✗ Branch 5 → 20 not taken.
✗ Branch 6 → 7 not taken.
✗ Branch 6 → 18 not taken.
✗ Branch 7 → 8 not taken.
✗ Branch 7 → 16 not taken.
✓ Branch 12 → 13 taken 14 times.
✗ Branch 12 → 24 not taken.
 32 INSTANTIATE_TEST_SUITE_P(, BenchmarkTests, ::testing::ValuesIn(TestUtil::collectTestCases("benchmark", false)),
638 TestUtil::NameResolver());
639
640 class ExampleTests : public ::testing::TestWithParam<TestCase> {};
641
7/16
✓ Branch 6 → 7 taken 2 times.
✗ Branch 6 → 53 not taken.
✓ Branch 10 → 11 taken 2 times.
✗ Branch 10 → 45 not taken.
✓ Branch 11 → 12 taken 2 times.
✗ Branch 11 → 43 not taken.
✓ Branch 14 → 15 taken 2 times.
✗ Branch 14 → 38 not taken.
✓ Branch 16 → 17 taken 2 times.
✗ Branch 16 → 34 not taken.
✓ Branch 18 → 19 taken 2 times.
✗ Branch 18 → 31 not taken.
✗ Branch 19 → 20 not taken.
✓ Branch 19 → 21 taken 2 times.
✗ Branch 31 → 32 not taken.
✗ Branch 31 → 33 not taken.
 34 TEST_P(ExampleTests, ) { runTestCase(GetParam()); }
642
4/14
spice::testing::gtest_ExampleTests_EvalGenerator_():
✓ Branch 2 → 3 taken 2 times.
✗ Branch 2 → 10 not taken.
✓ Branch 3 → 4 taken 2 times.
✗ Branch 3 → 8 not taken.
spice::testing::gtest_ExampleTests_EvalGenerateName_(testing::TestParamInfo<spice::testing::TestCase> const&):
✗ Branch 3 → 4 not taken.
✓ Branch 3 → 12 taken 10 times.
✗ Branch 5 → 6 not taken.
✗ Branch 5 → 20 not taken.
✗ Branch 6 → 7 not taken.
✗ Branch 6 → 18 not taken.
✗ Branch 7 → 8 not taken.
✗ Branch 7 → 16 not taken.
✓ Branch 12 → 13 taken 10 times.
✗ Branch 12 → 24 not taken.
 24 INSTANTIATE_TEST_SUITE_P(, ExampleTests, ::testing::ValuesIn(TestUtil::collectTestCases("examples", false)),
643 TestUtil::NameResolver());
644
645 class LinterTests : public ::testing::TestWithParam<TestCase> {};
646
7/16
✓ Branch 6 → 7 taken 2 times.
✗ Branch 6 → 53 not taken.
✓ Branch 10 → 11 taken 2 times.
✗ Branch 10 → 45 not taken.
✓ Branch 11 → 12 taken 2 times.
✗ Branch 11 → 43 not taken.
✓ Branch 14 → 15 taken 2 times.
✗ Branch 14 → 38 not taken.
✓ Branch 16 → 17 taken 2 times.
✗ Branch 16 → 34 not taken.
✓ Branch 18 → 19 taken 2 times.
✗ Branch 18 → 31 not taken.
✗ Branch 19 → 20 not taken.
✓ Branch 19 → 21 taken 2 times.
✗ Branch 31 → 32 not taken.
✗ Branch 31 → 33 not taken.
 26 TEST_P(LinterTests, ) {
647
1/2
✓ Branch 2 → 3 taken 6 times.
✗ Branch 2 → 10 not taken.
 6 if (!testDriverCliOptions.bootstrapMode)
648
4/8
✓ Branch 3 → 4 taken 6 times.
✗ Branch 3 → 18 not taken.
✓ Branch 4 → 5 taken 6 times.
✗ Branch 4 → 16 not taken.
✓ Branch 5 → 6 taken 6 times.
✗ Branch 5 → 15 not taken.
✓ Branch 6 → 7 taken 6 times.
✗ Branch 6 → 13 not taken.
 6 GTEST_SKIP() << "Linter tests run against the bootstrap compiler only";
649 ✗ execBootstrapLinterTestCase(GetParam());
650 }
651
4/14
spice::testing::gtest_LinterTests_EvalGenerator_():
✓ Branch 2 → 3 taken 2 times.
✗ Branch 2 → 10 not taken.
✓ Branch 3 → 4 taken 2 times.
✗ Branch 3 → 8 not taken.
spice::testing::gtest_LinterTests_EvalGenerateName_(testing::TestParamInfo<spice::testing::TestCase> const&):
✗ Branch 3 → 4 not taken.
✓ Branch 3 → 12 taken 6 times.
✗ Branch 5 → 6 not taken.
✗ Branch 5 → 20 not taken.
✗ Branch 6 → 7 not taken.
✗ Branch 6 → 18 not taken.
✗ Branch 7 → 8 not taken.
✗ Branch 7 → 16 not taken.
✓ Branch 12 → 13 taken 6 times.
✗ Branch 12 → 24 not taken.
 16 INSTANTIATE_TEST_SUITE_P(, LinterTests, ::testing::ValuesIn(TestUtil::collectTestCases("linter", false)),
652 TestUtil::NameResolver());
653
654 } // namespace spice::testing
655