GCC Code Coverage Report

Line	Branch	Exec	Source
1			// Copyright (c) 2021-2025 ChilliBits. All rights reserved.
2
3			// GCOV_EXCL_START
4
5			#include <string>
6			#include <vector>
7
8			#include <gtest/gtest.h>
9
10			#include <SourceFile.h>
11			#include <driver/Driver.h>
12			#include <exception/CompilerError.h>
13			#include <exception/LexerError.h>
14			#include <exception/LinkerError.h>
15			#include <exception/ParserError.h>
16			#include <exception/SemanticError.h>
17			#include <global/GlobalResourceManager.h>
18			#include <global/TypeRegistry.h>
19			#include <llvm/TargetParser/Host.h>
20			#include <llvm/TargetParser/Triple.h>
21			#include <symboltablebuilder/SymbolTable.h>
22			#include <util/FileUtil.h>
23
24			#include "driver/Driver.h"
25			#include "util/TestUtil.h"
26
27			using namespace spice::compiler;
28
29			namespace spice::testing {
30
31			extern TestDriverCliOptions testDriverCliOptions;
32
33		−	void execTestCase(const TestCase &testCase) {
34			// Check if test is disabled
35		−	if (TestUtil::isDisabled(testCase))
36		−	GTEST_SKIP();
37
38			// Create fake cli options
39		−	const llvm::Triple targetTriple(llvm::Triple::normalize(llvm::sys::getDefaultTargetTriple()));
40		−	CliOptions cliOptions = {
41		−	/* mainSourceFile= */ testCase.testPath / REF_NAME_SOURCE,
42			/* targetTriple= */ targetTriple,
43		−	/* targetArch= */ std::string(targetTriple.getArchName()),
44		−	/* targetVendor= */ std::string(targetTriple.getVendorName()),
45		−	/* targetOs= */ std::string(targetTriple.getOSName()),
46			/* isNativeTarget= */ true,
47			/* useCPUFeatures*/ false, // Disabled because it makes the refs differ on different machines
48			/* execute= */ false, // If we set this to 'true', the compiler will not emit object files
49			/* cacheDir= */ "./cache",
50			/* outputDir= */ "./",
51			/* outputPath= */ "",
52			/* buildMode= */ BuildMode::DEBUG,
53			/* compileJobCount= */ 0,
54			/* ignoreCache */ true,
55		−	/* llvmArgs= */ "",
56			/* printDebugOutput= */ false,
57			CliOptions::DumpSettings{
58			/* dumpCST= */ false,
59			/* dumpAST= */ false,
60			/* dumpSymbolTables= */ false,
61			/* dumpTypes= */ false,
62			/* dumpCacheStats= */ false,
63			/* dumpDependencyGraph= */ false,
64			/* dumpIR= */ false,
65			/* dumpAssembly= */ false,
66			/* dumpObjectFile= */ false,
67			/* dumpToFiles= */ false,
68			/* abortAfterDump */ false,
69			},
70			/* namesForIRValues= */ true,
71			/* useLifetimeMarkers= */ false,
72			/* optLevel= */ OptLevel::O0,
73		−	/* useLTO= */ exists(testCase.testPath / CTL_LTO),
74		−	/* noEntryFct= */ exists(testCase.testPath / CTL_RUN_BUILTIN_TESTS),
75		−	/* generateTestMain= */ exists(testCase.testPath / CTL_RUN_BUILTIN_TESTS),
76			/* staticLinking= */ false,
77			CliOptions::InstrumentationSettings{
78		−	/* generateDebugInfo= */ exists(testCase.testPath / CTL_DEBUG_INFO),
79			/* sanitizer= */ Sanitizer::NONE,
80			},
81			/* disableVerifier= */ false,
82			/* testMode= */ true,
83			/* comparableOutput= */ true,
84		−	};
85			static_assert(sizeof(CliOptions::DumpSettings) == 11, "CliOptions::DumpSettings struct size changed");
86			static_assert(sizeof(CliOptions::InstrumentationSettings) == 2, "CliOptions::InstrumentationSettings struct size changed");
87			static_assert(sizeof(CliOptions) == 384, "CliOptions struct size changed");
88
89			// Instantiate GlobalResourceManager
90		−	GlobalResourceManager resourceManager(cliOptions);
91
92			try {
93			// Create source file instance for main source file
94		−	SourceFile *mainSourceFile = resourceManager.createSourceFile(nullptr, MAIN_FILE_NAME, cliOptions.mainSourceFile, false);
95
96			// Run Lexer and Parser
97		−	mainSourceFile->runLexer();
98		−	mainSourceFile->runParser();
99
100			// Check CST
101		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_PARSE_TREE, [&] {
102		−	mainSourceFile->runCSTVisualizer();
103		−	return mainSourceFile->compilerOutput.cstString;
104			});
105
106			// Build and optimize AST
107		−	mainSourceFile->runASTBuilder();
108
109			// Check AST
110		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_SYNTAX_TREE, [&] {
111		−	mainSourceFile->runASTVisualizer();
112		−	return mainSourceFile->compilerOutput.astString;
113			});
114
115			// Execute import collector and semantic analysis stages
116		−	mainSourceFile->runImportCollector();
117		−	mainSourceFile->runSymbolTableBuilder();
118		−	mainSourceFile->runMiddleEnd(); // TypeChecker pre + post
119
120			// Check symbol table output (check happens here to include updates from type checker)
121		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_SYMBOL_TABLE,
122		−	[&] { return mainSourceFile->globalScope->getSymbolTableJSON().dump(/*indent=*/2); });
123
124			// Fail if an error was expected
125		−	if (exists(testCase.testPath / REF_NAME_ERROR_OUTPUT))
126		−	FAIL() << "Expected error, but got no error";
127
128			// Check dependency graph
129		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_DEP_GRAPH, [&] {
130		−	mainSourceFile->runDependencyGraphVisualizer();
131		−	return mainSourceFile->compilerOutput.depGraphString;
132			});
133
134			// Run backend for all dependencies
135		−	for (SourceFile *sourceFile : mainSourceFile->dependencies | std::views::values)
136		−	sourceFile->runBackEnd();
137
138			// Execute IR generator in normal or debug mode
139		−	mainSourceFile->runIRGenerator();
140
141			// Enable sanitizers
142		−	const std::filesystem::path sanitizerFile = testCase.testPath / INPUT_NAME_SANITIZER;
143		−	if (exists(sanitizerFile)) {
144		−	std::ifstream file(sanitizerFile, std::ios::binary);
145		−	std::ostringstream buffer;
146		−	buffer << file.rdbuf();
147		−	const std::string sanitizerString = buffer.str();
148		−	if (sanitizerString == SANITIZER_ADDRESS)
149		−	cliOptions.instrumentation.sanitizer = Sanitizer::ADDRESS;
150		−	else if (sanitizerString == SANITIZER_THREAD)
151		−	cliOptions.instrumentation.sanitizer = Sanitizer::THREAD;
152		−	}
153
154			// Check IR code
155		−	for (uint8_t i = 0; i <= 5; i++) {
156		−	TestUtil::checkRefMatch(
157		−	testCase.testPath / REF_NAME_OPT_IR[i],
158		−	[&] {
159		−	cliOptions.optLevel = static_cast<OptLevel>(i);
160
161		−	if (cliOptions.useLTO) {
162		−	mainSourceFile->runPreLinkIROptimizer();
163		−	mainSourceFile->runBitcodeLinker();
164		−	mainSourceFile->runPostLinkIROptimizer();
165			} else {
166		−	mainSourceFile->runDefaultIROptimizer();
167			}
168
169		−	return mainSourceFile->compilerOutput.irOptString;
170			},
171		−	[&](std::string &expectedOutput, std::string &actualOutput) {
172		−	if (cliOptions.instrumentation.generateDebugInfo) {
173			// Remove the lines, containing paths on the local file system
174		−	TestUtil::eraseLinesBySubstring(expectedOutput, " = !DIFile(filename:");
175		−	TestUtil::eraseLinesBySubstring(actualOutput, " = !DIFile(filename:");
176			}
177		−	},
178			true);
179			}
180
181			// Link the bitcode if not happened yet
182		−	if (cliOptions.useLTO && cliOptions.optLevel == OptLevel::O0)
183		−	mainSourceFile->runBitcodeLinker();
184
185			// Check assembly code (only when not running test on GitHub Actions)
186		−	bool objectFilesEmitted = false;
187		−	if (!testDriverCliOptions.isGitHubActions) {
188		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_ASM, [&] {
189		−	mainSourceFile->runObjectEmitter();
190		−	objectFilesEmitted = true;
191
192		−	return mainSourceFile->compilerOutput.asmString;
193			});
194			}
195
196			// Check warnings
197		−	mainSourceFile->collectAndPrintWarnings();
198		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_WARNING_OUTPUT, [&] {
199		−	std::stringstream actualWarningString;
200		−	for (const CompilerWarning &warning : mainSourceFile->compilerOutput.warnings)
201		−	actualWarningString << warning.warningMessage << "\n";
202		−	return actualWarningString.str();
203		−	});
204
205			// Do linking and conclude compilation
206		−	const bool needsNormalRun = TestUtil::doesRefExist(testCase.testPath / REF_NAME_EXECUTION_OUTPUT);
207		−	const bool needsDebuggerRun = TestUtil::doesRefExist(testCase.testPath / REF_NAME_GDB_OUTPUT);
208		−	if (needsNormalRun || needsDebuggerRun) {
209			// Emit main source file object if not done already
210		−	if (!objectFilesEmitted)
211		−	mainSourceFile->runObjectEmitter();
212
213			// Prepare linker
214		−	resourceManager.linker.outputPath = TestUtil::getDefaultExecutableName();
215
216			// Parse linker flags
217		−	const std::filesystem::path linkerFlagsFile = testCase.testPath / INPUT_NAME_LINKER_FLAGS;
218		−	if (exists(linkerFlagsFile))
219		−	for (const std::string &linkerFlag : TestUtil::getFileContentLinesVector(linkerFlagsFile))
220		−	resourceManager.linker.addLinkerFlag(linkerFlag);
221
222			// Conclude the compilation
223		−	mainSourceFile->concludeCompilation();
224
225			// Prepare and run linker
226		−	resourceManager.linker.prepare();
227		−	resourceManager.linker.link();
228		−	}
229
230			// Check type registry output
231		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_TYPE_REGISTRY, [&] { return TypeRegistry::dump(); });
232
233			// Check cache stats output
234		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_CACHE_STATS, [&] {
235		−	std::stringstream cacheStats;
236		−	cacheStats << FunctionManager::dumpLookupCacheStatistics() << std::endl;
237		−	cacheStats << StructManager::dumpLookupCacheStatistics() << std::endl;
238		−	cacheStats << InterfaceManager::dumpLookupCacheStatistics() << std::endl;
239		−	return cacheStats.str();
240		−	});
241
242			// Check if the execution output matches the expected output
243		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_EXECUTION_OUTPUT, [&] {
244		−	const std::filesystem::path cliFlagsFile = testCase.testPath / INPUT_NAME_CLI_FLAGS;
245			// Execute binary
246		−	std::stringstream cmd;
247		−	if (testDriverCliOptions.enableLeakDetection)
248		−	cmd << "valgrind -q --leak-check=full --num-callers=100 --error-exitcode=1 ";
249		−	cmd << TestUtil::getDefaultExecutableName();
250		−	if (exists(cliFlagsFile))
251		−	cmd << " " << TestUtil::getFileContentLinesVector(cliFlagsFile).at(0);
252		−	const auto [output, exitCode] = FileUtil::exec(cmd.str(), true);
253
254			#if not OS_WINDOWS // Windows does not give us the exit code, so we cannot check it on Windows
255			// Check if the exit code matches the expected one
256			// If no exit code ref file exists, check against 0
257		−	if (TestUtil::checkRefMatch(testCase.testPath / REF_NAME_EXIT_CODE, [&] { return std::to_string(exitCode); })) {
258		−	EXPECT_NE(0, exitCode) << "Program exited with zero exit code, but expected erroneous exit code";
259			} else {
260		−	EXPECT_EQ(0, exitCode) << "Program exited with non-zero exit code";
261			}
262			#endif
263
264		−	return output;
265		−	});
266
267			// Check if the debugger output matches the expected output
268		−	if (!testDriverCliOptions.isGitHubActions) { // GDB tests are currently not support on GH actions
269		−	TestUtil::checkRefMatch(
270		−	testCase.testPath / REF_NAME_GDB_OUTPUT,
271		−	[&] {
272			// Execute debugger script
273		−	std::filesystem::path gdbScriptPath = testCase.testPath / CTL_DEBUG_SCRIPT;
274		−	EXPECT_TRUE(std::filesystem::exists(gdbScriptPath)) << "Debug output requested, but debug script not found";
275		−	gdbScriptPath.make_preferred();
276		−	const std::string cmd = "gdb -x " + gdbScriptPath.string() + " " + TestUtil::getDefaultExecutableName();
277		−	const auto [output, exitCode] = FileUtil::exec(cmd);
278
279			#if not OS_WINDOWS // Windows does not give us the exit code, so we cannot check it on Windows
280		−	EXPECT_EQ(0, exitCode) << "GDB exited with non-zero exit code when running debug script";
281			#endif
282
283		−	return output;
284		−	},
285		−	[&](std::string &expectedOutput, std::string &actualOutput) {
286			// Do not compare against the GDB header
287		−	TestUtil::eraseGDBHeader(expectedOutput);
288		−	TestUtil::eraseGDBHeader(actualOutput);
289		−	});
290			}
291		−	} catch (LexerError &error) {
292		−	TestUtil::handleError(testCase, error);
293		−	} catch (ParserError &error) {
294		−	TestUtil::handleError(testCase, error);
295		−	} catch (SemanticError &error) {
296		−	TestUtil::handleError(testCase, error);
297		−	} catch (CompilerError &error) {
298		−	TestUtil::handleError(testCase, error);
299		−	} catch (LinkerError &error) {
300		−	TestUtil::handleError(testCase, error);
301		−	} catch (std::exception &error) {
302		−	TestUtil::handleError(testCase, error);
303		−	}
304
305		−	SUCCEED();
306		−	}
307
308			class CommonTests : public ::testing::TestWithParam<TestCase> {};
309		−	TEST_P(CommonTests, ) { execTestCase(GetParam()); }
310		−	INSTANTIATE_TEST_SUITE_P(, CommonTests, ::testing::ValuesIn(TestUtil::collectTestCases("common", false)),
311			TestUtil::NameResolver());
312
313			class LexerTests : public ::testing::TestWithParam<TestCase> {};
314		−	TEST_P(LexerTests, ) { execTestCase(GetParam()); }
315		−	INSTANTIATE_TEST_SUITE_P(, LexerTests, ::testing::ValuesIn(TestUtil::collectTestCases("lexer", false)), TestUtil::NameResolver());
316
317			class ParserTests : public ::testing::TestWithParam<TestCase> {};
318		−	TEST_P(ParserTests, ) { execTestCase(GetParam()); }
319		−	INSTANTIATE_TEST_SUITE_P(, ParserTests, ::testing::ValuesIn(TestUtil::collectTestCases("parser", false)),
320			TestUtil::NameResolver());
321
322			class SymbolTableBuilderTests : public ::testing::TestWithParam<TestCase> {};
323		−	TEST_P(SymbolTableBuilderTests, ) { execTestCase(GetParam()); }
324		−	INSTANTIATE_TEST_SUITE_P(, SymbolTableBuilderTests, ::testing::ValuesIn(TestUtil::collectTestCases("symboltablebuilder", true)),
325			TestUtil::NameResolver());
326
327			class TypeCheckerTests : public ::testing::TestWithParam<TestCase> {};
328		−	TEST_P(TypeCheckerTests, ) { execTestCase(GetParam()); }
329		−	INSTANTIATE_TEST_SUITE_P(, TypeCheckerTests, ::testing::ValuesIn(TestUtil::collectTestCases("typechecker", true)),
330			TestUtil::NameResolver());
331
332			class IRGeneratorTests : public ::testing::TestWithParam<TestCase> {};
333		−	TEST_P(IRGeneratorTests, ) { execTestCase(GetParam()); }
334		−	INSTANTIATE_TEST_SUITE_P(, IRGeneratorTests, ::testing::ValuesIn(TestUtil::collectTestCases("irgenerator", true)),
335			TestUtil::NameResolver());
336
337			class StdTests : public ::testing::TestWithParam<TestCase> {};
338		−	TEST_P(StdTests, ) { execTestCase(GetParam()); }
339		−	INSTANTIATE_TEST_SUITE_P(, StdTests, ::testing::ValuesIn(TestUtil::collectTestCases("std", true)), TestUtil::NameResolver());
340
341			class BenchmarkTests : public ::testing::TestWithParam<TestCase> {};
342		−	TEST_P(BenchmarkTests, ) { execTestCase(GetParam()); }
343		−	INSTANTIATE_TEST_SUITE_P(, BenchmarkTests, ::testing::ValuesIn(TestUtil::collectTestCases("benchmark", false)),
344			TestUtil::NameResolver());
345
346			class ExampleTests : public ::testing::TestWithParam<TestCase> {};
347		−	TEST_P(ExampleTests, ) { execTestCase(GetParam()); }
348		−	INSTANTIATE_TEST_SUITE_P(, ExampleTests, ::testing::ValuesIn(TestUtil::collectTestCases("examples", false)),
349			TestUtil::NameResolver());
350
351			class BootstrapCompilerTests : public ::testing::TestWithParam<TestCase> {};
352		−	TEST_P(BootstrapCompilerTests, ) { execTestCase(GetParam()); }
353		−	INSTANTIATE_TEST_SUITE_P(, BootstrapCompilerTests, ::testing::ValuesIn(TestUtil::collectTestCases("bootstrap-compiler", false)),
354			TestUtil::NameResolver());
355
356			} // namespace spice::testing
357
358			// GCOV_EXCL_STOP
359