GCC Code Coverage Report

Directory: ../
File: test/TestRunner.cpp
Date: 2024-11-22 23:10:59
Exec Total Coverage
Lines: 0 0 100.0%
Functions: 0 0 -%
Branches: 0 0 -%
Line Branch Exec Source
1 // Copyright (c) 2021-2024 ChilliBits. All rights reserved.
2
3 // GCOV_EXCL_START
4
5 #include <string>
6 #include <vector>
7
8 #include <gtest/gtest.h>
9
10 #include <llvm/TargetParser/Host.h>
11 #include <llvm/TargetParser/Triple.h>
12
13 #include <SourceFile.h>
14 #include <driver/Driver.h>
15 #include <exception/CompilerError.h>
16 #include <exception/LexerError.h>
17 #include <exception/LinkerError.h>
18 #include <exception/ParserError.h>
19 #include <exception/SemanticError.h>
20 #include <global/GlobalResourceManager.h>
21 #include <global/TypeRegistry.h>
22 #include <symboltablebuilder/SymbolTable.h>
23 #include <util/FileUtil.h>
24
25 #include "util/TestUtil.h"
26
27 using namespace spice::compiler;
28
29 namespace spice::testing {
30
31 − void execTestCase(const TestCase &testCase) {
32 // Check if test is disabled
33 − if (TestUtil::isDisabled(testCase, skipNonGitHubTests))
34 − GTEST_SKIP();
35
36 // Create fake cli options
37 − const std::filesystem::path sourceFilePath = testCase.testPath / REF_NAME_SOURCE;
38 − const llvm::Triple targetTriple(llvm::Triple::normalize(llvm::sys::getDefaultTargetTriple()));
39 − CliOptions cliOptions = {
40 /* mainSourceFile= */ sourceFilePath,
41 − /* targetTriple= */ targetTriple.getTriple(),
42 − /* targetArch= */ std::string(targetTriple.getArchName()),
43 − /* targetVendor= */ std::string(targetTriple.getVendorName()),
44 − /* targetOs= */ std::string(targetTriple.getOSName()),
45 /* isNativeTarget= */ true,
46 /* useCPUFeatures*/ false, // Disabled because it makes the refs differ on different machines
47 /* execute= */ false, // If we set this to 'true', the compiler will not emit object files
48 /* cacheDir= */ "./cache",
49 /* outputDir= */ "./",
50 /* outputPath= */ "",
51 /* buildMode= */ DEBUG,
52 /* compileJobCount= */ 0,
53 /* ignoreCache */ true,
54 /* llvmArgs= */ "",
55 /* printDebugOutput= */ false,
56 CliOptions::DumpSettings{
57 /* dumpCST= */ false,
58 /* dumpAST= */ false,
59 /* dumpSymbolTables= */ false,
60 /* dumpTypes= */ false,
61 /* dumpIR= */ false,
62 /* dumpAssembly= */ false,
63 /* dumpObjectFile= */ false,
64 /* dumpToFiles= */ false,
65 /* abortAfterDump */ false,
66 },
67 /* namesForIRValues= */ true,
68 /* useLifetimeMarkers= */ false,
69 /* optLevel= */ O0,
70 − /* useLTO= */ exists(testCase.testPath / CTL_LTO),
71 − /* noEntryFct= */ exists(testCase.testPath / CTL_RUN_BUILTIN_TESTS),
72 − /* generateTestMain= */ exists(testCase.testPath / CTL_RUN_BUILTIN_TESTS),
73 /* staticLinking= */ false,
74 − /* debugInfo= */ exists(testCase.testPath / CTL_DEBUG_INFO),
75 /* disableVerifier= */ false,
76 /* testMode= */ true,
77 − };
78 static_assert(sizeof(CliOptions::DumpSettings) == 9, "CliOptions::DumpSettings struct size changed");
79 static_assert(sizeof(CliOptions) == 360, "CliOptions struct size changed");
80
81 // Instantiate GlobalResourceManager
82 − GlobalResourceManager resourceManager(cliOptions);
83
84 try {
85 // Create source file instance for main source file
86 − SourceFile *mainSourceFile = resourceManager.createSourceFile(nullptr, MAIN_FILE_NAME, cliOptions.mainSourceFile, false);
87
88 // Run Lexer and Parser
89 − mainSourceFile->runLexer();
90 − mainSourceFile->runParser();
91
92 // Check CST
93 − TestUtil::checkRefMatch(testCase.testPath / REF_NAME_PARSE_TREE, [&] {
94 − mainSourceFile->runCSTVisualizer();
95 − return mainSourceFile->compilerOutput.cstString;
96 });
97
98 // Build and optimize AST
99 − mainSourceFile->runASTBuilder();
100
101 // Check AST
102 − TestUtil::checkRefMatch(testCase.testPath / REF_NAME_SYNTAX_TREE, [&] {
103 − mainSourceFile->runASTVisualizer();
104 − return mainSourceFile->compilerOutput.astString;
105 });
106
107 // Execute import collector and semantic analysis stages
108 − mainSourceFile->runImportCollector();
109 − mainSourceFile->runSymbolTableBuilder();
110 − mainSourceFile->runTypeChecker();
111
112 // Check symbol table output (check happens here to include updates from type checker)
113 − TestUtil::checkRefMatch(testCase.testPath / REF_NAME_SYMBOL_TABLE,
114 − [&] { return mainSourceFile->globalScope->getSymbolTableJSON().dump(/*indent=*/2); });
115
116 // Fail if an error was expected
117 − if (exists(testCase.testPath / REF_NAME_ERROR_OUTPUT))
118 − FAIL() << "Expected error, but got no error";
119
120 // Run backend for all dependencies
121 − for (const auto &[name, sourceFile] : mainSourceFile->dependencies)
122 − sourceFile->runBackEnd();
123
124 // Execute IR generator in normal or debug mode
125 − mainSourceFile->runIRGenerator();
126
127 // Check unoptimized IR code
128 − TestUtil::checkRefMatch(
129 − testCase.testPath / REF_NAME_IR, [&] { return mainSourceFile->compilerOutput.irString; },
130 − [&](std::string &expectedOutput, std::string &actualOutput) {
131 − if (cliOptions.generateDebugInfo) {
132 // Remove the lines, containing paths on the local file system
133 − TestUtil::eraseLinesBySubstring(expectedOutput, " = !DIFile(filename:");
134 − TestUtil::eraseLinesBySubstring(actualOutput, " = !DIFile(filename:");
135 }
136 − });
137
138 // Check optimized IR code
139 − for (uint8_t i = 1; i <= 5; i++) {
140 − TestUtil::checkRefMatch(testCase.testPath / REF_NAME_OPT_IR[i - 1], [&] {
141 − cliOptions.optLevel = static_cast<OptLevel>(i);
142
143 − if (cliOptions.useLTO) {
144 − mainSourceFile->runPreLinkIROptimizer();
145 − mainSourceFile->runBitcodeLinker();
146 − mainSourceFile->runPostLinkIROptimizer();
147 } else {
148 − mainSourceFile->runDefaultIROptimizer();
149 }
150
151 − return mainSourceFile->compilerOutput.irOptString;
152 });
153 }
154
155 // Link the bitcode if not happened yet
156 − if (cliOptions.useLTO && cliOptions.optLevel == O0)
157 − mainSourceFile->runBitcodeLinker();
158
159 // Check assembly code
160 − bool objectFilesEmitted = false;
161 − if (!skipNonGitHubTests) {
162 − TestUtil::checkRefMatch(testCase.testPath / REF_NAME_ASM, [&] {
163 − mainSourceFile->runObjectEmitter();
164 − objectFilesEmitted = true;
165
166 − return mainSourceFile->compilerOutput.asmString;
167 });
168 }
169
170 // Check warnings
171 − mainSourceFile->collectAndPrintWarnings();
172 − TestUtil::checkRefMatch(testCase.testPath / REF_NAME_WARNING_OUTPUT, [&] {
173 − std::stringstream actualWarningString;
174 − for (const CompilerWarning &warning : mainSourceFile->compilerOutput.warnings)
175 − actualWarningString << warning.warningMessage << "\n";
176 − return actualWarningString.str();
177 − });
178
179 // Do linking and conclude compilation
180 − const bool needsNormalRun = exists(testCase.testPath / REF_NAME_EXECUTION_OUTPUT);
181 − const bool needsDebuggerRun = exists(testCase.testPath / REF_NAME_GDB_OUTPUT);
182 − if (needsNormalRun || needsDebuggerRun) {
183 // Prepare linker
184 − resourceManager.linker.outputPath = TestUtil::getDefaultExecutableName();
185
186 // Parse linker flags
187 − const std::filesystem::path linkerFlagsFile = testCase.testPath / INPUT_NAME_LINKER_FLAGS;
188 − if (exists(linkerFlagsFile))
189 − for (const std::string &linkerFlag : TestUtil::getFileContentLinesVector(linkerFlagsFile))
190 − resourceManager.linker.addLinkerFlag(linkerFlag);
191
192 // Emit main source file object if not done already
193 − if (!objectFilesEmitted)
194 − mainSourceFile->runObjectEmitter();
195
196 // Conclude the compilation
197 − mainSourceFile->concludeCompilation();
198
199 // Prepare and run linker
200 − resourceManager.linker.prepare();
201 − resourceManager.linker.link();
202 − }
203
204 // Check type registry output
205 − TestUtil::checkRefMatch(testCase.testPath / REF_NAME_TYPE_REGISTRY, [&] { return TypeRegistry::dump(); });
206
207 // Check if the execution output matches the expected output
208 − TestUtil::checkRefMatch(testCase.testPath / REF_NAME_EXECUTION_OUTPUT, [&] {
209 − const std::filesystem::path cliFlagsFile = testCase.testPath / INPUT_NAME_CLI_FLAGS;
210 // Execute binary
211 − std::stringstream cmd;
212 − if (enableLeakDetection)
213 − cmd << "valgrind -q --leak-check=full --num-callers=100 --error-exitcode=1 ";
214 − cmd << TestUtil::getDefaultExecutableName();
215 − if (exists(cliFlagsFile))
216 − cmd << " " << TestUtil::getFileContentLinesVector(cliFlagsFile).at(0);
217 − const auto [output, exitCode] = FileUtil::exec(cmd.str());
218
219 #if not OS_WINDOWS // Windows does not give us the exit code, so we cannot check it on Windows
220 // Check if the exit code matches the expected one
221 // If no exit code ref file exists, check against 0
222 − if (TestUtil::checkRefMatch(testCase.testPath / REF_NAME_EXIT_CODE, [&] { return std::to_string(exitCode); })) {
223 − EXPECT_NE(0, exitCode) << "Program exited with zero exit code, but expected erronous exit code";
224 } else {
225 − EXPECT_EQ(0, exitCode) << "Program exited with non-zero exit code";
226 }
227 #endif
228
229 − return output;
230 − });
231
232 // Check if the debugger output matches the expected output
233 − if (!skipNonGitHubTests) { // GDB tests are currently not support on GH actions
234 − TestUtil::checkRefMatch(
235 − testCase.testPath / REF_NAME_GDB_OUTPUT,
236 − [&] {
237 // Execute debugger script
238 − std::filesystem::path gdbScriptPath = testCase.testPath / CTL_DEBUG_SCRIPT;
239 − EXPECT_TRUE(std::filesystem::exists(gdbScriptPath)) << "Debug output requested, but debug script not found";
240 − gdbScriptPath.make_preferred();
241 − const std::string cmd = "gdb -x " + gdbScriptPath.string() + " " + TestUtil::getDefaultExecutableName();
242 − const auto [output, exitCode] = FileUtil::exec(cmd);
243
244 #if not OS_WINDOWS // Windows does not give us the exit code, so we cannot check it on Windows
245 − EXPECT_EQ(0, exitCode) << "GDB exited with non-zero exit code when running debug script";
246 #endif
247
248 − return output;
249 − },
250 − [&](std::string &expectedOutput, std::string &actualOutput) {
251 // Do not compare against the GDB header
252 − TestUtil::eraseGDBHeader(expectedOutput);
253 − TestUtil::eraseGDBHeader(actualOutput);
254 − });
255 }
256 − } catch (LexerError &error) {
257 − TestUtil::handleError(testCase, error);
258 − } catch (ParserError &error) {
259 − TestUtil::handleError(testCase, error);
260 − } catch (SemanticError &error) {
261 − TestUtil::handleError(testCase, error);
262 − } catch (CompilerError &error) {
263 − TestUtil::handleError(testCase, error);
264 − } catch (LinkerError &error) {
265 − TestUtil::handleError(testCase, error);
266 − } catch (std::exception &error) {
267 − TestUtil::handleError(testCase, error);
268 − }
269
270 − SUCCEED();
271 − }
272
273 class CommonTests : public ::testing::TestWithParam<TestCase> {};
274 − TEST_P(CommonTests, ) { execTestCase(GetParam()); }
275 − INSTANTIATE_TEST_SUITE_P(, CommonTests, ::testing::ValuesIn(TestUtil::collectTestCases("common", false)),
276 TestUtil::NameResolver());
277
278 class LexerTests : public ::testing::TestWithParam<TestCase> {};
279 − TEST_P(LexerTests, ) { execTestCase(GetParam()); }
280 − INSTANTIATE_TEST_SUITE_P(, LexerTests, ::testing::ValuesIn(TestUtil::collectTestCases("lexer", false)), TestUtil::NameResolver());
281
282 class ParserTests : public ::testing::TestWithParam<TestCase> {};
283 − TEST_P(ParserTests, ) { execTestCase(GetParam()); }
284 − INSTANTIATE_TEST_SUITE_P(, ParserTests, ::testing::ValuesIn(TestUtil::collectTestCases("parser", false)),
285 TestUtil::NameResolver());
286
287 class SymbolTableBuilderTests : public ::testing::TestWithParam<TestCase> {};
288 − TEST_P(SymbolTableBuilderTests, ) { execTestCase(GetParam()); }
289 − INSTANTIATE_TEST_SUITE_P(, SymbolTableBuilderTests, ::testing::ValuesIn(TestUtil::collectTestCases("symboltablebuilder", true)),
290 TestUtil::NameResolver());
291
292 class TypeCheckerTests : public ::testing::TestWithParam<TestCase> {};
293 − TEST_P(TypeCheckerTests, ) { execTestCase(GetParam()); }
294 − INSTANTIATE_TEST_SUITE_P(, TypeCheckerTests, ::testing::ValuesIn(TestUtil::collectTestCases("typechecker", true)),
295 TestUtil::NameResolver());
296
297 class IRGeneratorTests : public ::testing::TestWithParam<TestCase> {};
298 − TEST_P(IRGeneratorTests, ) { execTestCase(GetParam()); }
299 − INSTANTIATE_TEST_SUITE_P(, IRGeneratorTests, ::testing::ValuesIn(TestUtil::collectTestCases("irgenerator", true)),
300 TestUtil::NameResolver());
301
302 class StdTests : public ::testing::TestWithParam<TestCase> {};
303 − TEST_P(StdTests, ) { execTestCase(GetParam()); }
304 − INSTANTIATE_TEST_SUITE_P(, StdTests, ::testing::ValuesIn(TestUtil::collectTestCases("std", true)), TestUtil::NameResolver());
305
306 class BenchmarkTests : public ::testing::TestWithParam<TestCase> {};
307 − TEST_P(BenchmarkTests, ) { execTestCase(GetParam()); }
308 − INSTANTIATE_TEST_SUITE_P(, BenchmarkTests, ::testing::ValuesIn(TestUtil::collectTestCases("benchmark", false)),
309 TestUtil::NameResolver());
310
311 class ExampleTests : public ::testing::TestWithParam<TestCase> {};
312 − TEST_P(ExampleTests, ) { execTestCase(GetParam()); }
313 − INSTANTIATE_TEST_SUITE_P(, ExampleTests, ::testing::ValuesIn(TestUtil::collectTestCases("examples", false)),
314 TestUtil::NameResolver());
315
316 class BootstrapCompilerTests : public ::testing::TestWithParam<TestCase> {};
317 − TEST_P(BootstrapCompilerTests, ) { execTestCase(GetParam()); }
318 − INSTANTIATE_TEST_SUITE_P(, BootstrapCompilerTests, ::testing::ValuesIn(TestUtil::collectTestCases("bootstrap-compiler", false)),
319 TestUtil::NameResolver());
320
321 } // namespace spice::testing
322
323 // GCOV_EXCL_STOP
324