linker/external-linker-interface.spice
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // Copyright (c) 2021-2026 ChilliBits. All rights reserved. | ||
| 2 | |||
| 3 | // Std imports | ||
| 4 | import "std/data/vector"; | ||
| 5 | import "std/io/filepath"; | ||
| 6 | import "std/io/file"; | ||
| 7 | import "std/data/unordered-map"; | ||
| 8 | import "std/os/mutex"; | ||
| 9 | import "std/os/thread"; | ||
| 10 | import "std/type/type-conversion"; | ||
| 11 | |||
| 12 | // Own imports | ||
| 13 | import "@/exception/fatal-error"; | ||
| 14 | import "@/driver"; | ||
| 15 | import "@/util/system-util"; | ||
| 16 | import "@/util/compile-progress"; | ||
| 17 | import "@/exception/compiler-error"; | ||
| 18 | import "@/exception/linker-error"; | ||
| 19 | |||
| 20 | /** | ||
| 21 | * Compilation of an additional source to an object file | ||
| 22 | */ | ||
| 23 | 20 | type AdditionalSourceCompilation struct { | |
| 24 | FilePath additionalSource | ||
| 25 | String linkerInvokerPath | ||
| 26 | Vector<String> args | ||
| 27 | FilePath objectFilePath | ||
| 28 | ExecResult result | ||
| 29 | } | ||
| 30 | |||
| 31 | ✗ | public type ExternalLinkerInterface struct { | |
| 32 | CliOptions& cliOptions | ||
| 33 | // The following members are public to be observable from tests | ||
| 34 | public Vector<FilePath> linkedFiles | ||
| 35 | public Vector<String> linkerFlags | ||
| 36 | public FilePath outputPath | ||
| 37 | 1055 | public bool linkLibMath = false | |
| 38 | Mutex linkLibMathMutex // The IR generator requests libm linkage from the worker threads of the parallel back end | ||
| 39 | Vector<FilePath> additionalSourcePaths | ||
| 40 | // Object files, that the compiler emitted itself. Only these are intermediary, so only these get cleaned up after linking | ||
| 41 | Vector<FilePath> emittedObjectFiles | ||
| 42 | // Expanding a linker flag may run commands (backtick substitutions), so every flag is only expanded once | ||
| 43 | UnorderedMap<String, Vector<String>> expandedLinkerFlags | ||
| 44 | // Compilations of additional sources, that were started early to overlap with the back end | ||
| 45 | Vector<AdditionalSourceCompilation> additionalSourceCompilations | ||
| 46 | Vector<Thread> additionalSourceCompilationThreads | ||
| 47 | } | ||
| 48 | |||
| 49 | 1055 | public p ExternalLinkerInterface.ctor(CliOptions& cliOptions) { | |
| 50 | 1055 | this.cliOptions = cliOptions; | |
| 51 | 1055 | this.outputPath = cliOptions.outputPath; | |
| 52 | 1055 | } | |
| 53 | |||
| 54 | 766 | public p ExternalLinkerInterface.dtor() { | |
| 55 | // The compilation threads write into the members of this linker, so wait for them before it goes away. This is only | ||
| 56 | // relevant, if the compilation was aborted before the linker ran. | ||
| 57 | 766 | this.joinAdditionalSourceCompilationThreads(); | |
| 58 | 766 | } | |
| 59 | |||
| 60 | /** | ||
| 61 | * Prepare the linker for the linking process by assembling the required linker flags | ||
| 62 | */ | ||
| 63 | 462 | public p ExternalLinkerInterface.prepare() { | |
| 64 | // Static or shared linking | ||
| 65 | 462 | if this.cliOptions.outputContainer == OutputContainer::SHARED_LIBRARY { | |
| 66 | 1 | this.addLinkerFlag("-shared"); | |
| 67 | 462 | } else if this.cliOptions.staticLinking { | |
| 68 | // A static PIE on Linux, so that static executables get ASLR for their image as well. This needs all linked archives | ||
| 69 | // to be position independent (see setup-deps.py and the release workflow for the archives of the compiler itself) | ||
| 70 | ✗ | const bool staticPie = this.isTargetLinux() && this.cliOptions.outputContainer == OutputContainer::EXECUTABLE; | |
| 71 | ✗ | this.addLinkerFlag(staticPie ? "-static-pie" : "-static"); | |
| 72 | 461 | } | |
| 73 | |||
| 74 | // Windows linkers stamp the link time into the PE header. Leave it out, so that the same input always links to the same | ||
| 75 | // binary (reproducible builds, bootstrap fixed point) | ||
| 76 | 462 | if this.isTargetWindows() { | |
| 77 | ✗ | this.addLinkerFlag("-Wl,--no-insert-timestamp"); | |
| 78 | ✗ | } | |
| 79 | |||
| 80 | // Full RELRO on Linux: resolve all symbols at load time ('now') and remap the relocation sections, including the GOT, as | ||
| 81 | // read-only afterwards ('relro'). This takes away the GOT overwrite primitive of memory corruption exploits. | ||
| 82 | 462 | if this.isTargetLinux() && (this.cliOptions.outputContainer == OutputContainer::EXECUTABLE || this.cliOptions.outputContainer == OutputContainer::SHARED_LIBRARY) { | |
| 83 | 461 | this.addLinkerFlag("-Wl,-z,relro"); | |
| 84 | 461 | this.addLinkerFlag("-Wl,-z,now"); | |
| 85 | 461 | } | |
| 86 | |||
| 87 | // The following flags only make sense if we want to emit an executable | ||
| 88 | 462 | if this.cliOptions.outputContainer != OutputContainer::EXECUTABLE { | |
| 89 | 2 | return; | |
| 90 | } | ||
| 91 | |||
| 92 | // Position independent executable on Linux: the code is always generated position independent (see the 'PIE Level' module | ||
| 93 | // flag), so request a PIE explicitly, instead of relying on the default of the linker invoker (e.g. distributions, whose | ||
| 94 | // toolchain defaults to non-PIE). Static executables are linked with '-static-pie' above. | ||
| 95 | 460 | if this.isTargetLinux() && !this.cliOptions.staticLinking { | |
| 96 | 460 | this.addLinkerFlag("-pie"); | |
| 97 | 460 | } | |
| 98 | |||
| 99 | // Stripping symbols, on request only: the symbol table is what a run-time symbolizer such as the stack trace runtime | ||
| 100 | // resolves addresses against, so a stripped executable prints nothing but addresses. Darwin is excluded, because its | ||
| 101 | // linker has no equivalent of '-Wl,-s'. | ||
| 102 | 460 | if this.cliOptions.stripSymbols && !this.isTargetDarwin() { | |
| 103 | ✗ | this.addLinkerFlag("-Wl,-s"); | |
| 104 | ✗ | } | |
| 105 | |||
| 106 | // Sanitizers | ||
| 107 | 460 | switch this.cliOptions.instrumentation.sanitizer { | |
| 108 | case Sanitizer::ADDRESS: { | ||
| 109 | 13 | this.addLinkerFlag("-fsanitize=address"); | |
| 110 | 13 | } | |
| 111 | case Sanitizer::THREAD: { | ||
| 112 | 2 | this.addLinkerFlag("-fsanitize=thread"); | |
| 113 | 2 | } | |
| 114 | case Sanitizer::MEMORY: { | ||
| 115 | ✗ | this.addLinkerFlag("-fsanitize=memory"); | |
| 116 | ✗ | this.requestLibMathLinkage(); | |
| 117 | ✗ | } | |
| 118 | case Sanitizer::TYPE: { | ||
| 119 | ✗ | this.addLinkerFlag("-fsanitize=type"); | |
| 120 | ✗ | } | |
| 121 | } | ||
| 122 | |||
| 123 | // Code coverage | ||
| 124 | 460 | if this.cliOptions.instrumentation.codeCoverage { | |
| 125 | ✗ | this.addLinkerFlag("--coverage"); | |
| 126 | ✗ | } | |
| 127 | |||
| 128 | // Web Assembly | ||
| 129 | 460 | if this.isTargetWasm() { | |
| 130 | ✗ | this.addLinkerFlag("-nostdlib"); | |
| 131 | ✗ | this.addLinkerFlag("-Wl,--no-entry"); | |
| 132 | ✗ | this.addLinkerFlag("-Wl,--export-all"); | |
| 133 | ✗ | } | |
| 134 | 460 | } | |
| 135 | |||
| 136 | /** | ||
| 137 | * Run the linker or archiver, depending on the requested output container | ||
| 138 | */ | ||
| 139 | 462 | public p ExternalLinkerInterface.run() { | |
| 140 | 462 | switch this.cliOptions.outputContainer { | |
| 141 | case OutputContainer::EXECUTABLE, OutputContainer::SHARED_LIBRARY: { | ||
| 142 | 461 | this.link(); | |
| 143 | 461 | } | |
| 144 | case OutputContainer::STATIC_LIBRARY: { | ||
| 145 | 1 | this.archive(); | |
| 146 | 1 | } | |
| 147 | case OutputContainer::OBJECT_FILE: { | ||
| 148 | // No linking necessary | ||
| 149 | ✗ | } | |
| 150 | default: { | ||
| 151 | ✗ | panic(Error("Unknown output container")); | |
| 152 | } | ||
| 153 | } | ||
| 154 | 462 | } | |
| 155 | |||
| 156 | /** | ||
| 157 | * Cleanup intermediary object files | ||
| 158 | */ | ||
| 159 | 462 | public p ExternalLinkerInterface.cleanup() { | |
| 160 | // Wait for additional source compilations, that are still running (e.g. because the linker did not need them) | ||
| 161 | 462 | this.joinAdditionalSourceCompilationThreads(); | |
| 162 | |||
| 163 | // Cleanup intermediary object files | ||
| 164 | 462 | if this.cliOptions.outputContainer == OutputContainer::OBJECT_FILE || this.cliOptions.dump.dumpToFiles { | |
| 165 | ✗ | return; | |
| 166 | } | ||
| 167 | 467 | foreach const AdditionalSourceCompilation& compilation : this.additionalSourceCompilations { | |
| 168 | 5 | if compilation.objectFilePath.exists() { | |
| 169 | 5 | const bool deleted = deleteFile(compilation.objectFilePath.toString()); | |
| 170 | 5 | assert deleted; | |
| 171 | 5 | } | |
| 172 | 5 | } | |
| 173 | 3233 | foreach const FilePath& path : this.emittedObjectFiles { | |
| 174 | 2771 | const bool deleted = deleteFile(path.toString()); | |
| 175 | 2771 | assert deleted; | |
| 176 | 2771 | } | |
| 177 | 462 | } | |
| 178 | |||
| 179 | /** | ||
| 180 | * Link the object files to an executable or shared library | ||
| 181 | */ | ||
| 182 | 461 | p ExternalLinkerInterface.link() { | |
| 183 | 461 | assert !this.outputPath.isEmpty(); | |
| 184 | |||
| 185 | // Find the linker invoker and linker | ||
| 186 | 461 | const ExternalBinaryFinderResult linkerInvoker = findLinkerInvoker(); | |
| 187 | 461 | const ExternalBinaryFinderResult linker = findLinker(this.cliOptions); | |
| 188 | 461 | const bool isGccInvoker = linkerInvoker.name == "gcc"; | |
| 189 | 461 | const bool isClangInvoker = linkerInvoker.name == "clang"; | |
| 190 | // GCC does not implement MemorySanitizer or TypeSanitizer | ||
| 191 | 461 | const Sanitizer sanitizer = this.cliOptions.instrumentation.sanitizer; | |
| 192 | 461 | if !isClangInvoker && (sanitizer == Sanitizer::MEMORY || sanitizer == Sanitizer::TYPE) { | |
| 193 | ✗ | const string msg = "Memory and type sanitizers require clang as the linker invoker, but 'gcc' was selected"; | |
| 194 | ✗ | const LinkerError error = LinkerError(LinkerErrorType::SANITIZER_NOT_SUPPORTED_BY_LINKER_INVOKER, msg); | |
| 195 | ✗ | abortWithError(error.getMessage()); | |
| 196 | ✗ | } | |
| 197 | |||
| 198 | // Build the linker argument vector. Each entry is passed verbatim to the linker invoker (no shell), so file paths can | ||
| 199 | // never be re-interpreted as shell syntax - this is what prevents command injection via attacker-controlled object-file | ||
| 200 | // or output paths. | ||
| 201 | 461 | Vector<String> args = this.getInvokerArgs(linker, isGccInvoker); | |
| 202 | // Append output path | ||
| 203 | 461 | args.pushBack(String("-o")); | |
| 204 | 461 | args.pushBack(String(this.outputPath.toString())); | |
| 205 | // Append object files first, so that '-l' archive flags added below are searched for symbols that are already | ||
| 206 | // undefined at the point they appear in the command. Additional sources are compiled to object files first, which | ||
| 207 | // mostly happened in the background. | ||
| 208 | 461 | this.joinAdditionalSourceCompilationThreads(); | |
| 209 | 3236 | foreach const FilePath& objectFilePath : this.linkedFiles { | |
| 210 | 2775 | if this.isAdditionalSource(objectFilePath) { | |
| 211 | 5 | const FilePath compiledObjectFilePath = this.compileAdditionalSource(objectFilePath); | |
| 212 | 5 | args.pushBack(String(compiledObjectFilePath.toString())); | |
| 213 | 5 | } else { | |
| 214 | 2770 | args.pushBack(String(objectFilePath.toString())); | |
| 215 | 2770 | } | |
| 216 | 2775 | } | |
| 217 | // Append the std's runtime library search path and the linker flags | ||
| 218 | 461 | const Vector<String> trailingArgs = this.getTrailingArgs(); | |
| 219 | 2802 | foreach const String& trailingArg : trailingArgs { | |
| 220 | 2341 | args.pushBack(trailingArg); | |
| 221 | 2341 | } | |
| 222 | 461 | if this.linkLibMath { | |
| 223 | 2 | args.pushBack(String("-lm")); | |
| 224 | 2 | } | |
| 225 | |||
| 226 | // Print status message | ||
| 227 | 461 | if this.cliOptions.printDebugOutput { | |
| 228 | ✗ | const String command = renderCommandForDisplay(linkerInvoker.path, args); | |
| 229 | ✗ | printf("\nLinking with: %s (invoker) / %s (linker)", linkerInvoker.name, linker.name); | |
| 230 | ✗ | printf("\nLinker command: %s", command.getRaw()); | |
| 231 | ✗ | printf("\nEmitting executable to path: %s\n", this.outputPath.toString()); | |
| 232 | ✗ | } | |
| 233 | |||
| 234 | // Call the linker | ||
| 235 | 461 | this.runTool(linkerInvoker.path, args, "Linker", "Linking"); | |
| 236 | 461 | } | |
| 237 | |||
| 238 | /** | ||
| 239 | * Get the leading arguments for the linker invoker, that select the linker and the target | ||
| 240 | * | ||
| 241 | * @param linker Linker | ||
| 242 | * @param isGccInvoker Whether the linker invoker is GCC | ||
| 243 | * @return Leading arguments | ||
| 244 | */ | ||
| 245 | 471 | f<Vector<String>> ExternalLinkerInterface.getInvokerArgs(const ExternalBinaryFinderResult& linker, bool isGccInvoker) { | |
| 246 | 471 | Vector<String> args; | |
| 247 | // GCC 16 dropped '-fuse-ld=ld'; skip when using GCC with the default BFD linker | ||
| 248 | 471 | if !isGccInvoker || linker.name != "ld" { | |
| 249 | 471 | args.pushBack("-fuse-ld=" + linker.path); | |
| 250 | 471 | } | |
| 251 | // '--target=' is clang-only; GCC uses target-specific toolchain prefixes instead | ||
| 252 | 471 | if !isGccInvoker { | |
| 253 | 471 | args.pushBack(String("--target=") + this.cliOptions.targetTriple.value); | |
| 254 | 471 | } | |
| 255 | 471 | return args; | |
| 256 | } | ||
| 257 | |||
| 258 | /** | ||
| 259 | * Get the trailing arguments for the linker invoker: the library search path of the std and the expanded linker flags | ||
| 260 | * | ||
| 261 | * @return Trailing arguments | ||
| 262 | */ | ||
| 263 | 471 | f<Vector<String>> ExternalLinkerInterface.getTrailingArgs() { | |
| 264 | 471 | Vector<String> args; | |
| 265 | // Append the std's own library search path, so the runtime modules can link against the static support libraries that | ||
| 266 | // ship with the std (currently std/runtime/lib/libbacktrace.a). It goes ahead of the linker flags below, because search | ||
| 267 | // directories are tried in the order given. Passed verbatim rather than through expandLinkerFlag(), so that a std path | ||
| 268 | // containing '$' or a backtick is not mistaken for a variable reference or a command substitution. Only for a native | ||
| 269 | // build: those archives are built for the machine the std was installed on, so offering them while linking for another | ||
| 270 | // target can only ever produce a mismatch. | ||
| 271 | 471 | if this.cliOptions.isNativeTarget { | |
| 272 | 471 | const FilePath stdRuntimeLibDir = getStdRuntimeLibDir(); | |
| 273 | 471 | if !stdRuntimeLibDir.isEmpty() { | |
| 274 | 471 | args.pushBack(String("-L") + stdRuntimeLibDir.toString()); | |
| 275 | 471 | } | |
| 276 | 471 | } | |
| 277 | // Append linker flags, expanding any environment-variable references or backtick command substitutions they may | ||
| 278 | // contain (e.g. std bindings using "-L$LLVM_LIB_DIR" or "`pkg-config --cflags --libs libcurl`"); a single flag can | ||
| 279 | // expand into several argv entries. They go behind the object files (libraries must follow the objects that reference | ||
| 280 | // them). A static link gets the static flags from pkg-config. | ||
| 281 | 3456 | foreach const String& linkerFlag : this.linkerFlags { | |
| 282 | 2985 | if !this.expandedLinkerFlags.contains(linkerFlag) { | |
| 283 | 1854 | this.expandedLinkerFlags.upsert(linkerFlag, expandLinkerFlag(linkerFlag, this.cliOptions.staticLinking)); | |
| 284 | 1854 | } | |
| 285 | 2985 | const Vector<String>& expandedFlags = this.expandedLinkerFlags.get(linkerFlag); | |
| 286 | 6016 | foreach const String& expandedFlag : expandedFlags { | |
| 287 | 3031 | args.pushBack(expandedFlag); | |
| 288 | 3031 | } | |
| 289 | 2985 | } | |
| 290 | |||
| 291 | 471 | return args; | |
| 292 | } | ||
| 293 | |||
| 294 | /** | ||
| 295 | * Get the arguments for the linker invoker to compile an additional source to an object file. These are the arguments, that | ||
| 296 | * the linker invoker would use to compile the additional source as part of the link command, without the ones that only | ||
| 297 | * affect linking. | ||
| 298 | * | ||
| 299 | * @param additionalSource Additional source file | ||
| 300 | * @param objectFilePath Path of the object file to emit | ||
| 301 | * @return Compile arguments | ||
| 302 | */ | ||
| 303 | 10 | f<Vector<String>> ExternalLinkerInterface.getAdditionalSourceCompileArgs(const FilePath& additionalSource, const FilePath& objectFilePath) { | |
| 304 | 10 | const ExternalBinaryFinderResult linkerInvoker = findLinkerInvoker(); | |
| 305 | 10 | const ExternalBinaryFinderResult linker = findLinker(this.cliOptions); | |
| 306 | 10 | const bool isGccInvoker = linkerInvoker.name == "gcc"; | |
| 307 | 10 | Vector<String> args; | |
| 308 | 10 | appendCompileArgs(args, this.getInvokerArgs(linker, isGccInvoker)); | |
| 309 | // Position independent code on Linux, because the object file is linked into a PIE or a shared library | ||
| 310 | 10 | if this.isTargetLinux() { | |
| 311 | 10 | args.pushBack(String("-fPIC")); | |
| 312 | 10 | } | |
| 313 | 10 | args.pushBack(String("-c")); | |
| 314 | 10 | args.pushBack(String("-o")); | |
| 315 | 10 | args.pushBack(String(objectFilePath.toString())); | |
| 316 | 10 | args.pushBack(String(additionalSource.toString())); | |
| 317 | 10 | appendCompileArgs(args, this.getTrailingArgs()); | |
| 318 | 20 | return args; | |
| 319 | 10 | } | |
| 320 | |||
| 321 | /** | ||
| 322 | * Append the given linker invoker arguments to the compile arguments of an additional source, except the ones that only | ||
| 323 | * affect linking. The linker invoker ignores those when compiling with '-c', but warns about each of them. Arguments, that | ||
| 324 | * affect compiling as well (e.g. '-I', '-D', '-pthread', '-fsanitize=...' or '--coverage'), are kept. | ||
| 325 | * | ||
| 326 | * @param compileArgs Compile arguments to append to | ||
| 327 | * @param invokerArgs Linker invoker arguments | ||
| 328 | */ | ||
| 329 | 21 | p appendCompileArgs(Vector<String>& compileArgs, const Vector<String>& invokerArgs) { | |
| 330 | 21 | bool skipNext = false; | |
| 331 | 1218 | foreach const String& arg : invokerArgs { | |
| 332 | 1197 | if skipNext { | |
| 333 | 1 | skipNext = false; | |
| 334 | 1 | continue; | |
| 335 | } | ||
| 336 | // '-Xlinker' passes the next argument on to the linker | ||
| 337 | 1196 | if arg == "-Xlinker" { | |
| 338 | 1 | skipNext = true; | |
| 339 | 1 | continue; | |
| 340 | } | ||
| 341 | 1195 | if !isLinkOnlyArg(arg) { | |
| 342 | 44 | compileArgs.pushBack(arg); | |
| 343 | 44 | } | |
| 344 | 1195 | } | |
| 345 | 21 | } | |
| 346 | |||
| 347 | /** | ||
| 348 | * Check if the given linker invoker argument only affects linking | ||
| 349 | * | ||
| 350 | * @param arg Linker invoker argument | ||
| 351 | * @return Link only or not | ||
| 352 | */ | ||
| 353 | 1195 | f<bool> isLinkOnlyArg(const String& arg) { | |
| 354 | // Linker inputs, like archives, shared libraries or object files | ||
| 355 | 1195 | if !arg.startsWith("-") { | |
| 356 | 21 | return true; | |
| 357 | } | ||
| 358 | // Libraries, library search paths, linker options and the linker selection | ||
| 359 | 1174 | if arg.startsWith("-l") || arg.startsWith("-L") || arg.startsWith("-Wl,") || arg.startsWith("-fuse-ld=") { | |
| 360 | 1122 | return true; | |
| 361 | } | ||
| 362 | 52 | const string[7] linkOnlyFlags = ["-shared", "-static", "-static-pie", "-pie", "-no-pie", "-rdynamic", "-nostdlib"]; | |
| 363 | 380 | for unsigned int i = 0; i < len(linkOnlyFlags); i++ { | |
| 364 | 336 | if arg == linkOnlyFlags[i] { | |
| 365 | 8 | return true; | |
| 366 | } | ||
| 367 | 328 | } | |
| 368 | 44 | return false; | |
| 369 | } | ||
| 370 | |||
| 371 | /** | ||
| 372 | * Get the path of the object file, an additional source is compiled to | ||
| 373 | * | ||
| 374 | * @param additionalSource Additional source file | ||
| 375 | * @return Object file path | ||
| 376 | */ | ||
| 377 | 10 | f<FilePath> ExternalLinkerInterface.getAdditionalSourceObjectFilePath(const FilePath& additionalSource) { | |
| 378 | // Additional sources with the same file name may come from different directories, so add the index of the source | ||
| 379 | 10 | unsigned long index = 0l; | |
| 380 | 12 | while index < this.additionalSourcePaths.getSize() && this.additionalSourcePaths.get(index) != additionalSource { | |
| 381 | 2 | index++; | |
| 382 | 2 | } | |
| 383 | 10 | const string objFileExt = getOutputFileExtension(this.cliOptions, OutputContainer::OBJECT_FILE); | |
| 384 | 10 | const String fileName = additionalSource.getFileName() + "." + toString(index) + "." + objFileExt; | |
| 385 | 20 | return this.cliOptions.outputDir / fileName; | |
| 386 | 10 | } | |
| 387 | |||
| 388 | /** | ||
| 389 | * Check if the given linked file is an additional source | ||
| 390 | * | ||
| 391 | * @param path Path of the linked file | ||
| 392 | * @return Additional source or not | ||
| 393 | */ | ||
| 394 | 2781 | f<bool> ExternalLinkerInterface.isAdditionalSource(const FilePath& path) { | |
| 395 | 2850 | foreach const FilePath& additionalSource : this.additionalSourcePaths { | |
| 396 | 74 | if additionalSource == path { | |
| 397 | 5 | return true; | |
| 398 | } | ||
| 399 | 69 | } | |
| 400 | 2776 | return false; | |
| 401 | } | ||
| 402 | |||
| 403 | /** | ||
| 404 | * Start compiling the additional sources to object files in the background, so that this overlaps with the back end. The | ||
| 405 | * linker waits for the results and only compiles an additional source again, if its compile arguments changed in the | ||
| 406 | * meantime. | ||
| 407 | */ | ||
| 408 | 462 | public p ExternalLinkerInterface.startAdditionalSourceCompilation() { | |
| 409 | 462 | if this.cliOptions.outputContainer != OutputContainer::EXECUTABLE && this.cliOptions.outputContainer != OutputContainer::SHARED_LIBRARY { | |
| 410 | 1 | return; | |
| 411 | } | ||
| 412 | 461 | if !this.additionalSourceCompilations.isEmpty() { | |
| 413 | ✗ | return; | |
| 414 | } | ||
| 415 | |||
| 416 | 461 | const ExternalBinaryFinderResult linkerInvoker = findLinkerInvoker(); | |
| 417 | 466 | foreach const FilePath& additionalSource : this.additionalSourcePaths { | |
| 418 | 5 | AdditionalSourceCompilation compilation; | |
| 419 | 5 | compilation.additionalSource = additionalSource; | |
| 420 | 5 | compilation.linkerInvokerPath = linkerInvoker.path; | |
| 421 | 5 | compilation.objectFilePath = this.getAdditionalSourceObjectFilePath(additionalSource); | |
| 422 | 5 | compilation.args = this.getAdditionalSourceCompileArgs(additionalSource, compilation.objectFilePath); | |
| 423 | 5 | this.additionalSourceCompilations.pushBack(compilation); | |
| 424 | 5 | } | |
| 425 | |||
| 426 | // Start the threads only after all compilations were added, so that no compilation is relocated while a thread works on it | ||
| 427 | 461 | this.additionalSourceCompilationThreads.reserve(this.additionalSourceCompilations.getSize()); | |
| 428 | 466 | for unsigned long i = 0l; i < this.additionalSourceCompilations.getSize(); i++ { | |
| 429 | 5 | AdditionalSourceCompilation* compilation = &this.additionalSourceCompilations.get(i); | |
| 430 | 10 | this.additionalSourceCompilationThreads.pushBack(Thread(p() [[async]] { | |
| 431 | 5 | compilation.result = exec(compilation.linkerInvokerPath, compilation.args, true); | |
| 432 | 5 | })); | |
| 433 | 5 | Thread& thread = this.additionalSourceCompilationThreads.back(); | |
| 434 | 5 | thread.run(); | |
| 435 | 5 | } | |
| 436 | 461 | } | |
| 437 | |||
| 438 | /** | ||
| 439 | * Wait for the additional source compilations, that run in the background | ||
| 440 | */ | ||
| 441 | 1689 | p ExternalLinkerInterface.joinAdditionalSourceCompilationThreads() { | |
| 442 | 1694 | foreach Thread& thread : this.additionalSourceCompilationThreads { | |
| 443 | 5 | thread.join(); | |
| 444 | 5 | } | |
| 445 | 1689 | this.additionalSourceCompilationThreads.clear(); | |
| 446 | 1689 | } | |
| 447 | |||
| 448 | /** | ||
| 449 | * Compile an additional source to an object file or take the result of the compilation, that ran in the background | ||
| 450 | * | ||
| 451 | * @param additionalSource Additional source file | ||
| 452 | * @return Object file path | ||
| 453 | */ | ||
| 454 | 5 | f<FilePath> ExternalLinkerInterface.compileAdditionalSource(const FilePath& additionalSource) { | |
| 455 | 5 | const FilePath objectFilePath = this.getAdditionalSourceObjectFilePath(additionalSource); | |
| 456 | 5 | const Vector<String> args = this.getAdditionalSourceCompileArgs(additionalSource, objectFilePath); | |
| 457 | |||
| 458 | // Compile it now, if it was not compiled in the background or with different arguments | ||
| 459 | 5 | AdditionalSourceCompilation* compilation = nil<AdditionalSourceCompilation*>; | |
| 460 | 6 | foreach AdditionalSourceCompilation& existingCompilation : this.additionalSourceCompilations { | |
| 461 | 6 | if existingCompilation.additionalSource == additionalSource { | |
| 462 | 5 | compilation = &existingCompilation; | |
| 463 | 5 | break; | |
| 464 | } | ||
| 465 | 1 | } | |
| 466 | 5 | if compilation == nil<AdditionalSourceCompilation*> || !areArgsEqual(compilation.args, args) { | |
| 467 | ✗ | if compilation == nil<AdditionalSourceCompilation*> { | |
| 468 | ✗ | AdditionalSourceCompilation newCompilation; | |
| 469 | ✗ | newCompilation.additionalSource = additionalSource; | |
| 470 | ✗ | this.additionalSourceCompilations.pushBack(newCompilation); | |
| 471 | ✗ | compilation = &this.additionalSourceCompilations.back(); | |
| 472 | ✗ | } | |
| 473 | ✗ | const ExternalBinaryFinderResult linkerInvoker = findLinkerInvoker(); | |
| 474 | // Assign via a reference: fields reached via a pointer might be uninitialized memory, so the compiler does not | ||
| 475 | // destruct their old values on assignment. The result of a background compilation would leak then | ||
| 476 | ✗ | AdditionalSourceCompilation& recompilation = *compilation; | |
| 477 | ✗ | recompilation.linkerInvokerPath = linkerInvoker.path; | |
| 478 | ✗ | recompilation.objectFilePath = objectFilePath; | |
| 479 | ✗ | recompilation.args = args; | |
| 480 | ✗ | recompilation.result = exec(recompilation.linkerInvokerPath, recompilation.args, true); | |
| 481 | ✗ | } | |
| 482 | |||
| 483 | // Check for compile error | ||
| 484 | 5 | if compilation.result.exitCode != 0 { | |
| 485 | ✗ | const String command = renderCommandForDisplay(compilation.linkerInvokerPath, compilation.args); | |
| 486 | ✗ | const String msg = "Compiling the additional source '" + additionalSource.toString() + "' failed\nCompile command: " + command + "\n" + compilation.result.output; | |
| 487 | ✗ | const LinkerError error = LinkerError(LinkerErrorType::LINKER_ERROR, msg.getRaw()); | |
| 488 | ✗ | abortWithError(error.getMessage()); | |
| 489 | ✗ | } | |
| 490 | 10 | return objectFilePath; | |
| 491 | 5 | } | |
| 492 | |||
| 493 | /** | ||
| 494 | * Check if two argument lists are equal. The equality operator of Vector compares the raw bytes of the items, which does not | ||
| 495 | * work for Strings. | ||
| 496 | * | ||
| 497 | * @param lhs First argument list | ||
| 498 | * @param rhs Second argument list | ||
| 499 | * @return Equal or not | ||
| 500 | */ | ||
| 501 | 5 | f<bool> areArgsEqual(const Vector<String>& lhs, const Vector<String>& rhs) { | |
| 502 | 5 | if lhs.getSize() != rhs.getSize() { | |
| 503 | ✗ | return false; | |
| 504 | } | ||
| 505 | 49 | for unsigned long i = 0l; i < lhs.getSize(); i++ { | |
| 506 | 44 | const String& lhsArg = lhs.get(i); | |
| 507 | 44 | const String& rhsArg = rhs.get(i); | |
| 508 | 44 | if lhsArg != rhsArg { | |
| 509 | ✗ | return false; | |
| 510 | } | ||
| 511 | 44 | } | |
| 512 | 5 | return true; | |
| 513 | } | ||
| 514 | |||
| 515 | /** | ||
| 516 | * Archive the object files to a static library | ||
| 517 | */ | ||
| 518 | 1 | p ExternalLinkerInterface.archive() { | |
| 519 | 1 | assert !this.outputPath.isEmpty(); | |
| 520 | |||
| 521 | // Find the archiver | ||
| 522 | 1 | const ExternalBinaryFinderResult archiver = findArchiver(); | |
| 523 | |||
| 524 | // Build the archiver argument vector (passed verbatim, no shell involved) | ||
| 525 | 1 | Vector<String> args; | |
| 526 | 1 | if archiver.name == "llvm-lib" || archiver.name == "lib" { | |
| 527 | // The archivers on Windows only understand the MSVC syntax | ||
| 528 | ✗ | args.pushBack(String("/OUT:") + this.outputPath.toString()); | |
| 529 | ✗ | } else { | |
| 530 | 1 | args.pushBack(String("rcs")); // r = insert files into archive; c = create archive if not existing; s = create archive index | |
| 531 | 1 | args.pushBack(String(this.outputPath.toString())); | |
| 532 | 1 | } | |
| 533 | 2 | foreach const FilePath& path : this.linkedFiles { | |
| 534 | 1 | args.pushBack(String(path.toString())); | |
| 535 | 1 | } | |
| 536 | |||
| 537 | // Print status message | ||
| 538 | 1 | if this.cliOptions.printDebugOutput { | |
| 539 | ✗ | const String command = renderCommandForDisplay(archiver.path, args); | |
| 540 | ✗ | printf("\nArchiving with: %s", archiver.name); | |
| 541 | ✗ | printf("\nArchiver command: %s", command.getRaw()); | |
| 542 | ✗ | printf("\nEmitting static library to path: %s\n", this.outputPath.toString()); | |
| 543 | ✗ | } | |
| 544 | |||
| 545 | // Call the archiver | ||
| 546 | 1 | this.runTool(archiver.path, args, "Archiver", "Archiving"); | |
| 547 | 1 | } | |
| 548 | |||
| 549 | /** | ||
| 550 | * Run the linker invoker or the archiver and abort with a linker error, if it fails | ||
| 551 | * | ||
| 552 | * @param toolPath Path to the tool | ||
| 553 | * @param args Arguments for the tool | ||
| 554 | * @param toolName Name of the tool for the error message, e.g. "Linker" | ||
| 555 | * @param activityName Name of the activity for the debug output, e.g. "Linking" | ||
| 556 | */ | ||
| 557 | 462 | p ExternalLinkerInterface.runTool(const String& toolPath, const Vector<String>& args, string toolName, string activityName) { | |
| 558 | // While the progress bar is shown, also capture the stderr of the tool, so that it does not end up behind the bar. It is | ||
| 559 | // printed after removing the bar, if the tool fails | ||
| 560 | 462 | const bool captureStdErr = isCompileProgressActive(); | |
| 561 | 462 | const ExecResult result = exec(toolPath, args, captureStdErr); | |
| 562 | |||
| 563 | // Check for error | ||
| 564 | 462 | if result.exitCode != 0 { | |
| 565 | ✗ | if captureStdErr { | |
| 566 | ✗ | printAboveActiveCompileProgress(result.output); | |
| 567 | ✗ | } | |
| 568 | ✗ | const String command = renderCommandForDisplay(toolPath, args); | |
| 569 | ✗ | const String msg = String(toolName) + " exited with non-zero exit code\n" + toolName + " command: " + command; | |
| 570 | ✗ | const LinkerError error = LinkerError(LinkerErrorType::LINKER_ERROR, msg.getRaw()); | |
| 571 | ✗ | abortWithError(error.getMessage()); | |
| 572 | ✗ | } | |
| 573 | |||
| 574 | // The captured output of a successful run (e.g. linker warnings) would be lost otherwise | ||
| 575 | 462 | if captureStdErr && !result.output.isEmpty() { | |
| 576 | ✗ | printAboveActiveCompileProgress(result.output); | |
| 577 | ✗ | } | |
| 578 | |||
| 579 | // Print the result if appropriate | ||
| 580 | 462 | if this.cliOptions.printDebugOutput && !result.output.isEmpty() { | |
| 581 | ✗ | printf("%s result: %s\n\n", activityName, result.output.getRaw()); | |
| 582 | ✗ | } | |
| 583 | 462 | } | |
| 584 | |||
| 585 | /** | ||
| 586 | * Add another object file to be linked when calling 'run()' | ||
| 587 | * | ||
| 588 | * @param path Path to the object file | ||
| 589 | */ | ||
| 590 | 3503 | public p ExternalLinkerInterface.addFileToLinkage(const FilePath& path) { | |
| 591 | // Skip files that are already registered | ||
| 592 | 21017 | foreach const FilePath& linkedFile : this.linkedFiles { | |
| 593 | 17514 | if linkedFile == path { | |
| 594 | ✗ | return; | |
| 595 | } | ||
| 596 | 17514 | } | |
| 597 | 3503 | this.linkedFiles.pushBack(path); | |
| 598 | 3503 | } | |
| 599 | |||
| 600 | /** | ||
| 601 | * Add an object file, that the compiler emitted itself, to be linked when calling 'run()'. Unlike other linked files, it is | ||
| 602 | * intermediary and gets deleted on cleanup. | ||
| 603 | * | ||
| 604 | * @param path Path to the emitted object file | ||
| 605 | */ | ||
| 606 | 3497 | public p ExternalLinkerInterface.addEmittedObjectFile(const FilePath& path) { | |
| 607 | 3497 | this.addFileToLinkage(path); | |
| 608 | // Source files with the same file name emit to the same object file path, which must only be deleted once | ||
| 609 | 20942 | foreach const FilePath& emittedObjectFile : this.emittedObjectFiles { | |
| 610 | 17445 | if emittedObjectFile == path { | |
| 611 | ✗ | return; | |
| 612 | } | ||
| 613 | 17445 | } | |
| 614 | 3497 | this.emittedObjectFiles.pushBack(path); | |
| 615 | 3497 | } | |
| 616 | |||
| 617 | /** | ||
| 618 | * Add another linker flag for the call to the linker executable | ||
| 619 | * | ||
| 620 | * @param linkerFlag Linker flag | ||
| 621 | */ | ||
| 622 | 1398 | public p ExternalLinkerInterface.addLinkerFlag(string linkerFlag) { | |
| 623 | 1398 | this.addLinkerFlag(String(linkerFlag)); | |
| 624 | 1398 | } | |
| 625 | |||
| 626 | /** | ||
| 627 | * Add another linker flag for the call to the linker executable | ||
| 628 | * | ||
| 629 | * @param linkerFlag Linker flag | ||
| 630 | */ | ||
| 631 | 1891 | public p ExternalLinkerInterface.addLinkerFlag(const String& linkerFlag) { | |
| 632 | // Group boundary markers must not be deduplicated: each archive group gets its own pair, and deduplication would | ||
| 633 | // collapse multiple groups into one misplaced pair. | ||
| 634 | 1891 | if linkerFlag != "-Wl,--start-group" && linkerFlag != "-Wl,--end-group" { | |
| 635 | 32412 | foreach const String& existingFlag : this.linkerFlags { | |
| 636 | 30587 | if existingFlag == linkerFlag { | |
| 637 | 6 | return; | |
| 638 | } | ||
| 639 | 30581 | } | |
| 640 | 1825 | } | |
| 641 | 1885 | this.linkerFlags.pushBack(linkerFlag); | |
| 642 | 1885 | } | |
| 643 | |||
| 644 | /** | ||
| 645 | * Add another source file to compile and link in (C or C++) | ||
| 646 | * | ||
| 647 | * @param additionalSource Additional source file | ||
| 648 | */ | ||
| 649 | 7 | public p ExternalLinkerInterface.addAdditionalSourcePath(const FilePath& additionalSource) { | |
| 650 | // Check if the file exists | ||
| 651 | 7 | if !additionalSource.exists() { | |
| 652 | 1 | const String msg = "The additional source file '" + additionalSource.toGenericString() + "' does not exist"; | |
| 653 | 1 | const CompilerError error = CompilerError(CompilerErrorType::IO_ERROR, msg.getRaw()); | |
| 654 | 1 | abortWithError(error.getMessage()); | |
| 655 | 1 | } | |
| 656 | |||
| 657 | // Add the file to the linker | ||
| 658 | 7 | const FilePath canonicalAdditionalSource = additionalSource.canonical(); | |
| 659 | 7 | this.addFileToLinkage(canonicalAdditionalSource); | |
| 660 | 7 | if !this.isAdditionalSource(canonicalAdditionalSource) { | |
| 661 | 6 | this.additionalSourcePaths.pushBack(canonicalAdditionalSource); | |
| 662 | 6 | } | |
| 663 | 7 | } | |
| 664 | |||
| 665 | /** | ||
| 666 | * Link against libmath a.k.a. -lm | ||
| 667 | */ | ||
| 668 | 3 | public p ExternalLinkerInterface.requestLibMathLinkage() { | |
| 669 | 3 | LockGuard _ = LockGuard(this.linkLibMathMutex); | |
| 670 | 3 | this.linkLibMath = true; | |
| 671 | 3 | } | |
| 672 | |||
| 673 | /** | ||
| 674 | * Check if the configured target is a WebAssembly target | ||
| 675 | * | ||
| 676 | * @return Whether the target is WebAssembly | ||
| 677 | */ | ||
| 678 | 460 | f<bool> ExternalLinkerInterface.isTargetWasm() { | |
| 679 | 460 | return this.cliOptions.targetTriple.isWasm(); | |
| 680 | } | ||
| 681 | |||
| 682 | /** | ||
| 683 | * Check if the configured target is a Linux based target | ||
| 684 | * | ||
| 685 | * @return Whether the target is Linux based | ||
| 686 | */ | ||
| 687 | 932 | f<bool> ExternalLinkerInterface.isTargetLinux() { | |
| 688 | 932 | return this.cliOptions.targetTriple.isOSLinux(); | |
| 689 | } | ||
| 690 | |||
| 691 | /** | ||
| 692 | * Check if the configured target is a Darwin (macOS) based target | ||
| 693 | * | ||
| 694 | * @return Whether the target is Darwin based | ||
| 695 | */ | ||
| 696 | − | f<bool> ExternalLinkerInterface.isTargetDarwin() { // LCOV_EXCL_LINE - coverage runs on x86_64 | |
| 697 | − | return this.cliOptions.targetTriple.isOSDarwin(); // LCOV_EXCL_LINE - coverage runs on x86_64 | |
| 698 | } // LCOV_EXCL_LINE - coverage runs on x86_64 | ||
| 699 | |||
| 700 | /** | ||
| 701 | * Check if the configured target is a Windows based target | ||
| 702 | * | ||
| 703 | * @return Whether the target is Windows based | ||
| 704 | */ | ||
| 705 | 462 | f<bool> ExternalLinkerInterface.isTargetWindows() { | |
| 706 | 462 | return this.cliOptions.targetTriple.isOSWindows(); | |
| 707 | } | ||
| 708 | |||
| 709 | 1 | #[test] | |
| 710 | f<bool> testAppendCompileArgs() { | ||
| 711 | 1 | Vector<String> invokerArgs; | |
| 712 | // Kept, because they affect compiling | ||
| 713 | 1 | invokerArgs.pushBack(String("--target=x86_64-unknown-linux-gnu")); | |
| 714 | 1 | invokerArgs.pushBack(String("-I/opt/llvm/include")); | |
| 715 | 1 | invokerArgs.pushBack(String("-DNDEBUG")); | |
| 716 | 1 | invokerArgs.pushBack(String("-pthread")); | |
| 717 | 1 | invokerArgs.pushBack(String("-fsanitize=address")); | |
| 718 | 1 | invokerArgs.pushBack(String("--coverage")); | |
| 719 | // Left out, because they only affect linking | ||
| 720 | 1 | invokerArgs.pushBack(String("-fuse-ld=/usr/bin/ld.lld")); | |
| 721 | 1 | invokerArgs.pushBack(String("-L/opt/llvm/lib")); | |
| 722 | 1 | invokerArgs.pushBack(String("-lLLVMCore")); | |
| 723 | 1 | invokerArgs.pushBack(String("-Wl,--start-group")); | |
| 724 | 1 | invokerArgs.pushBack(String("/opt/tpde/lib/libtpde.a")); | |
| 725 | 1 | invokerArgs.pushBack(String("-Xlinker")); | |
| 726 | 1 | invokerArgs.pushBack(String("--no-undefined")); | |
| 727 | 1 | invokerArgs.pushBack(String("-pie")); | |
| 728 | 1 | invokerArgs.pushBack(String("-static-pie")); | |
| 729 | 1 | invokerArgs.pushBack(String("-shared")); | |
| 730 | |||
| 731 | 1 | Vector<String> compileArgs; | |
| 732 | 1 | compileArgs.pushBack(String("-c")); | |
| 733 | 1 | appendCompileArgs(compileArgs, invokerArgs); | |
| 734 | 1 | assert compileArgs.getSize() == 7l; | |
| 735 | 1 | assert compileArgs.get(0l) == "-c"; | |
| 736 | 1 | assert compileArgs.get(1l) == "--target=x86_64-unknown-linux-gnu"; | |
| 737 | 1 | assert compileArgs.get(2l) == "-I/opt/llvm/include"; | |
| 738 | 1 | assert compileArgs.get(3l) == "-DNDEBUG"; | |
| 739 | 1 | assert compileArgs.get(4l) == "-pthread"; | |
| 740 | 1 | assert compileArgs.get(5l) == "-fsanitize=address"; | |
| 741 | 1 | assert compileArgs.get(6l) == "--coverage"; | |
| 742 | 1 | return true; | |
| 743 | 1 | } | |
| 744 |