ClangOffloadBundler.cpp source code [clang_source_code/tools/clang-offload-bundler/ClangOffloadBundler.cpp]

1	//===-- clang-offload-bundler/ClangOffloadBundler.cpp ---------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	///
9	/// \file
10	/// This file implements a clang-offload-bundler that bundles different
11	/// files that relate with the same source code but different targets into a
12	/// single one. Also the implements the opposite functionality, i.e. unbundle
13	/// files previous created by this tool.
14	///
15	//===----------------------------------------------------------------------===//
16
17	#include "clang/Basic/Version.h"
18	#include "llvm/ADT/ArrayRef.h"
19	#include "llvm/ADT/SmallString.h"
20	#include "llvm/ADT/StringMap.h"
21	#include "llvm/ADT/StringRef.h"
22	#include "llvm/ADT/StringSwitch.h"
23	#include "llvm/ADT/Triple.h"
24	#include "llvm/Bitcode/BitcodeWriter.h"
25	#include "llvm/IR/Constant.h"
26	#include "llvm/IR/Constants.h"
27	#include "llvm/IR/GlobalVariable.h"
28	#include "llvm/IR/LLVMContext.h"
29	#include "llvm/IR/Module.h"
30	#include "llvm/Object/Binary.h"
31	#include "llvm/Object/ObjectFile.h"
32	#include "llvm/Support/Casting.h"
33	#include "llvm/Support/CommandLine.h"
34	#include "llvm/Support/Error.h"
35	#include "llvm/Support/ErrorOr.h"
36	#include "llvm/Support/FileSystem.h"
37	#include "llvm/Support/MemoryBuffer.h"
38	#include "llvm/Support/Path.h"
39	#include "llvm/Support/Program.h"
40	#include "llvm/Support/raw_ostream.h"
41	#include "llvm/Support/Signals.h"
42	#include <algorithm>
43	#include <cassert>
44	#include <cstddef>
45	#include <cstdint>
46	#include <memory>
47	#include <string>
48	#include <system_error>
49	#include <vector>
50
51	using namespace llvm;
52	using namespace llvm::object;
53
54	static cl::opt<bool> Help("h", cl::desc("Alias for -help"), cl::Hidden);
55
56	// Mark all our options with this category, everything else (except for -version
57	// and -help) will be hidden.
58	static cl::OptionCategory
59	ClangOffloadBundlerCategory("clang-offload-bundler options");
60
61	static cl::list<std::string>
62	InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore,
63	cl::desc("[<input file>,...]"),
64	cl::cat(ClangOffloadBundlerCategory));
65	static cl::list<std::string>
66	OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore,
67	cl::desc("[<output file>,...]"),
68	cl::cat(ClangOffloadBundlerCategory));
69	static cl::list<std::string>
70	TargetNames("targets", cl::CommaSeparated, cl::OneOrMore,
71	cl::desc("[<offload kind>-<target triple>,...]"),
72	cl::cat(ClangOffloadBundlerCategory));
73	static cl::opt<std::string>
74	FilesType("type", cl::Required,
75	cl::desc("Type of the files to be bundled/unbundled.\n"
76	"Current supported types are:\n"
77	" i - cpp-output\n"
78	" ii - c++-cpp-output\n"
79	" ll - llvm\n"
80	" bc - llvm-bc\n"
81	" s - assembler\n"
82	" o - object\n"
83	" gch - precompiled-header\n"
84	" ast - clang AST file"),
85	cl::cat(ClangOffloadBundlerCategory));
86	static cl::opt<bool>
87	Unbundle("unbundle",
88	cl::desc("Unbundle bundled file into several output files.\n"),
89	cl::init(false), cl::cat(ClangOffloadBundlerCategory));
90
91	static cl::opt<bool> PrintExternalCommands(
92	"###",
93	cl::desc("Print any external commands that are to be executed "
94	"instead of actually executing them - for testing purposes.\n"),
95	cl::init(false), cl::cat(ClangOffloadBundlerCategory));
96
97	static cl::opt<bool> DumpTemporaryFiles(
98	"dump-temporary-files",
99	cl::desc("Dumps any temporary files created - for testing purposes.\n"),
100	cl::init(false), cl::cat(ClangOffloadBundlerCategory));
101
102	/// Magic string that marks the existence of offloading data.
103	#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__"
104
105	/// The index of the host input in the list of inputs.
106	static unsigned HostInputIndex = ~0u;
107
108	/// Path to the current binary.
109	static std::string BundlerExecutable;
110
111	/// Obtain the offload kind and real machine triple out of the target
112	/// information specified by the user.
113	static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind,
114	StringRef &Triple) {
115	auto KindTriplePair = Target.split('-');
116	OffloadKind = KindTriplePair.first;
117	Triple = KindTriplePair.second;
118	}
119	static StringRef getTriple(StringRef Target) {
120	StringRef OffloadKind;
121	StringRef Triple;
122	getOffloadKindAndTriple(Target, OffloadKind, Triple);
123	return Triple;
124	}
125	static bool hasHostKind(StringRef Target) {
126	StringRef OffloadKind;
127	StringRef Triple;
128	getOffloadKindAndTriple(Target, OffloadKind, Triple);
129	return OffloadKind == "host";
130	}
131
132	/// Generic file handler interface.
133	class FileHandler {
134	public:
135	FileHandler() {}
136
137	virtual ~FileHandler() {}
138
139	/// Update the file handler with information from the header of the bundled
140	/// file
141	virtual void ReadHeader(MemoryBuffer &Input) = 0;
142
143	/// Read the marker of the next bundled to be read in the file. The triple of
144	/// the target associated with that bundle is returned. An empty string is
145	/// returned if there are no more bundles to be read.
146	virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0;
147
148	/// Read the marker that closes the current bundle.
149	virtual void ReadBundleEnd(MemoryBuffer &Input) = 0;
150
151	/// Read the current bundle and write the result into the stream \a OS.
152	virtual void ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0;
153
154	/// Write the header of the bundled file to \a OS based on the information
155	/// gathered from \a Inputs.
156	virtual void WriteHeader(raw_fd_ostream &OS,
157	ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) = 0;
158
159	/// Write the marker that initiates a bundle for the triple \a TargetTriple to
160	/// \a OS.
161	virtual void WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) = 0;
162
163	/// Write the marker that closes a bundle for the triple \a TargetTriple to \a
164	/// OS. Return true if any error was found.
165
166	virtual bool WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) = 0;
167
168	/// Write the bundle from \a Input into \a OS.
169	virtual void WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0;
170	};
171
172	/// Handler for binary files. The bundled file will have the following format
173	/// (all integers are stored in little-endian format):
174	///
175	/// "OFFLOAD_BUNDLER_MAGIC_STR" (ASCII encoding of the string)
176	///
177	/// NumberOfOffloadBundles (8-byte integer)
178	///
179	/// OffsetOfBundle1 (8-byte integer)
180	/// SizeOfBundle1 (8-byte integer)
181	/// NumberOfBytesInTripleOfBundle1 (8-byte integer)
182	/// TripleOfBundle1 (byte length defined before)
183	///
184	/// ...
185	///
186	/// OffsetOfBundleN (8-byte integer)
187	/// SizeOfBundleN (8-byte integer)
188	/// NumberOfBytesInTripleOfBundleN (8-byte integer)
189	/// TripleOfBundleN (byte length defined before)
190	///
191	/// Bundle1
192	/// ...
193	/// BundleN
194
195	/// Read 8-byte integers from a buffer in little-endian format.
196	static uint64_t Read8byteIntegerFromBuffer(StringRef Buffer, size_t pos) {
197	uint64_t Res = 0;
198	const char *Data = Buffer.data();
199
200	for (unsigned i = 0; i < 8; ++i) {
201	Res <<= 8;
202	uint64_t Char = (uint64_t)Data[pos + 7 - i];
203	Res \|= 0xffu & Char;
204	}
205	return Res;
206	}
207
208	/// Write 8-byte integers to a buffer in little-endian format.
209	static void Write8byteIntegerToBuffer(raw_fd_ostream &OS, uint64_t Val) {
210	for (unsigned i = 0; i < 8; ++i) {
211	char Char = (char)(Val & 0xffu);
212	OS.write(&Char, 1);
213	Val >>= 8;
214	}
215	}
216
217	class BinaryFileHandler final : public FileHandler {
218	/// Information about the bundles extracted from the header.
219	struct BundleInfo final {
220	/// Size of the bundle.
221	uint64_t Size = 0u;
222	/// Offset at which the bundle starts in the bundled file.
223	uint64_t Offset = 0u;
224
225	BundleInfo() {}
226	BundleInfo(uint64_t Size, uint64_t Offset) : Size(Size), Offset(Offset) {}
227	};
228
229	/// Map between a triple and the corresponding bundle information.
230	StringMap<BundleInfo> BundlesInfo;
231
232	/// Iterator for the bundle information that is being read.
233	StringMap<BundleInfo>::iterator CurBundleInfo;
234
235	public:
236	BinaryFileHandler() : FileHandler() {}
237
238	~BinaryFileHandler() final {}
239
240	void ReadHeader(MemoryBuffer &Input) final {
241	StringRef FC = Input.getBuffer();
242
243	// Initialize the current bundle with the end of the container.
244	CurBundleInfo = BundlesInfo.end();
245
246	// Check if buffer is smaller than magic string.
247	size_t ReadChars = sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1;
248	if (ReadChars > FC.size())
249	return;
250
251	// Check if no magic was found.
252	StringRef Magic(FC.data(), sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1);
253	if (!Magic.equals(OFFLOAD_BUNDLER_MAGIC_STR))
254	return;
255
256	// Read number of bundles.
257	if (ReadChars + 8 > FC.size())
258	return;
259
260	uint64_t NumberOfBundles = Read8byteIntegerFromBuffer(FC, ReadChars);
261	ReadChars += 8;
262
263	// Read bundle offsets, sizes and triples.
264	for (uint64_t i = 0; i < NumberOfBundles; ++i) {
265
266	// Read offset.
267	if (ReadChars + 8 > FC.size())
268	return;
269
270	uint64_t Offset = Read8byteIntegerFromBuffer(FC, ReadChars);
271	ReadChars += 8;
272
273	// Read size.
274	if (ReadChars + 8 > FC.size())
275	return;
276
277	uint64_t Size = Read8byteIntegerFromBuffer(FC, ReadChars);
278	ReadChars += 8;
279
280	// Read triple size.
281	if (ReadChars + 8 > FC.size())
282	return;
283
284	uint64_t TripleSize = Read8byteIntegerFromBuffer(FC, ReadChars);
285	ReadChars += 8;
286
287	// Read triple.
288	if (ReadChars + TripleSize > FC.size())
289	return;
290
291	StringRef Triple(&FC.data()[ReadChars], TripleSize);
292	ReadChars += TripleSize;
293
294	// Check if the offset and size make sense.
295	if (!Offset \|\| Offset + Size > FC.size())
296	return;
297
298	(0) . __assert_fail ("BundlesInfo.find(Triple) == BundlesInfo.end() && \"Triple is duplicated??\"", "/home/seafit/code_projects/clang_source/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp", 299, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(BundlesInfo.find(Triple) == BundlesInfo.end() &&
299	(0) . __assert_fail ("BundlesInfo.find(Triple) == BundlesInfo.end() && \"Triple is duplicated??\"", "/home/seafit/code_projects/clang_source/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp", 299, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true"> "Triple is duplicated??");
300	BundlesInfo[Triple] = BundleInfo(Size, Offset);
301	}
302	// Set the iterator to where we will start to read.
303	CurBundleInfo = BundlesInfo.begin();
304	}
305
306	StringRef ReadBundleStart(MemoryBuffer &Input) final {
307	if (CurBundleInfo == BundlesInfo.end())
308	return StringRef();
309
310	return CurBundleInfo->first();
311	}
312
313	void ReadBundleEnd(MemoryBuffer &Input) final {
314	(0) . __assert_fail ("CurBundleInfo != BundlesInfo.end() && \"Invalid reader info!\"", "/home/seafit/code_projects/clang_source/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp", 314, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(CurBundleInfo != BundlesInfo.end() && "Invalid reader info!");
315	++CurBundleInfo;
316	}
317
318	void ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
319	(0) . __assert_fail ("CurBundleInfo != BundlesInfo.end() && \"Invalid reader info!\"", "/home/seafit/code_projects/clang_source/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp", 319, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(CurBundleInfo != BundlesInfo.end() && "Invalid reader info!");
320	StringRef FC = Input.getBuffer();
321	OS.write(FC.data() + CurBundleInfo->second.Offset,
322	CurBundleInfo->second.Size);
323	}
324
325	void WriteHeader(raw_fd_ostream &OS,
326	ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) final {
327	// Compute size of the header.
328	uint64_t HeaderSize = 0;
329
330	HeaderSize += sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1;
331	HeaderSize += 8; // Number of Bundles
332
333	for (auto &T : TargetNames) {
334	HeaderSize += 3 * 8; // Bundle offset, Size of bundle and size of triple.
335	HeaderSize += T.size(); // The triple.
336	}
337
338	// Write to the buffer the header.
339	OS << OFFLOAD_BUNDLER_MAGIC_STR;
340
341	Write8byteIntegerToBuffer(OS, TargetNames.size());
342
343	unsigned Idx = 0;
344	for (auto &T : TargetNames) {
345	MemoryBuffer &MB = *Inputs[Idx++].get();
346	// Bundle offset.
347	Write8byteIntegerToBuffer(OS, HeaderSize);
348	// Size of the bundle (adds to the next bundle's offset)
349	Write8byteIntegerToBuffer(OS, MB.getBufferSize());
350	HeaderSize += MB.getBufferSize();
351	// Size of the triple
352	Write8byteIntegerToBuffer(OS, T.size());
353	// Triple
354	OS << T;
355	}
356	}
357
358	void WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) final {}
359
360	bool WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) final {
361	return false;
362	}
363
364	void WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
365	OS.write(Input.getBufferStart(), Input.getBufferSize());
366	}
367	};
368
369	/// Handler for object files. The bundles are organized by sections with a
370	/// designated name.
371	///
372	/// In order to bundle we create an IR file with the content of each section and
373	/// use incremental linking to produce the resulting object. We also add section
374	/// with a single byte to state the name of the component the main object file
375	/// (the one we are bundling into) refers to.
376	///
377	/// To unbundle, we use just copy the contents of the designated section. If the
378	/// requested bundle refer to the main object file, we just copy it with no
379	/// changes.
380	class ObjectFileHandler final : public FileHandler {
381
382	/// The object file we are currently dealing with.
383	std::unique_ptr<ObjectFile> Obj;
384
385	/// Return the input file contents.
386	StringRef getInputFileContents() const { return Obj->getData(); }
387
388	/// Return true if the provided section is an offload section and return the
389	/// triple by reference.
390	static bool IsOffloadSection(SectionRef CurSection,
391	StringRef &OffloadTriple) {
392	StringRef SectionName;
393	CurSection.getName(SectionName);
394
395	if (SectionName.empty())
396	return false;
397
398	// If it does not start with the reserved suffix, just skip this section.
399	if (!SectionName.startswith(OFFLOAD_BUNDLER_MAGIC_STR))
400	return false;
401
402	// Return the triple that is right after the reserved prefix.
403	OffloadTriple = SectionName.substr(sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1);
404	return true;
405	}
406
407	/// Total number of inputs.
408	unsigned NumberOfInputs = 0;
409
410	/// Total number of processed inputs, i.e, inputs that were already
411	/// read from the buffers.
412	unsigned NumberOfProcessedInputs = 0;
413
414	/// LLVM context used to create the auxiliary modules.
415	LLVMContext VMContext;
416
417	/// LLVM module used to create an object with all the bundle
418	/// components.
419	std::unique_ptr<Module> AuxModule;
420
421	/// The current triple we are working with.
422	StringRef CurrentTriple;
423
424	/// The name of the main input file.
425	StringRef MainInputFileName;
426
427	/// Iterator of the current and next section.
428	section_iterator CurrentSection;
429	section_iterator NextSection;
430
431	public:
432	ObjectFileHandler(std::unique_ptr<ObjectFile> ObjIn)
433	: FileHandler(), Obj(std::move(ObjIn)),
434	CurrentSection(Obj->section_begin()),
435	NextSection(Obj->section_begin()) {}
436
437	~ObjectFileHandler() final {}
438
439	void ReadHeader(MemoryBuffer &Input) final {}
440
441	StringRef ReadBundleStart(MemoryBuffer &Input) final {
442	while (NextSection != Obj->section_end()) {
443	CurrentSection = NextSection;
444	++NextSection;
445
446	StringRef OffloadTriple;
447	// Check if the current section name starts with the reserved prefix. If
448	// so, return the triple.
449	if (IsOffloadSection(*CurrentSection, OffloadTriple))
450	return OffloadTriple;
451	}
452	return StringRef();
453	}
454
455	void ReadBundleEnd(MemoryBuffer &Input) final {}
456
457	void ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
458	// If the current section has size one, that means that the content we are
459	// interested in is the file itself. Otherwise it is the content of the
460	// section.
461	//
462	// TODO: Instead of copying the input file as is, deactivate the section
463	// that is no longer needed.
464
465	StringRef Content;
466	CurrentSection->getContents(Content);
467
468	if (Content.size() < 2)
469	OS.write(Input.getBufferStart(), Input.getBufferSize());
470	else
471	OS.write(Content.data(), Content.size());
472	}
473
474	void WriteHeader(raw_fd_ostream &OS,
475	ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) final {
476	(0) . __assert_fail ("HostInputIndex != ~0u && \"Host input index not defined.\"", "/home/seafit/code_projects/clang_source/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp", 476, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(HostInputIndex != ~0u && "Host input index not defined.");
477
478	// Record number of inputs.
479	NumberOfInputs = Inputs.size();
480
481	// Create an LLVM module to have the content we need to bundle.
482	auto *M = new Module("clang-offload-bundle", VMContext);
483	M->setTargetTriple(getTriple(TargetNames[HostInputIndex]));
484	AuxModule.reset(M);
485	}
486
487	void WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) final {
488	++NumberOfProcessedInputs;
489
490	// Record the triple we are using, that will be used to name the section we
491	// will create.
492	CurrentTriple = TargetTriple;
493	}
494
495	bool WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) final {
496	(0) . __assert_fail ("NumberOfProcessedInputs <= NumberOfInputs && \"Processing more inputs that actually exist!\"", "/home/seafit/code_projects/clang_source/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp", 497, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(NumberOfProcessedInputs <= NumberOfInputs &&
497	(0) . __assert_fail ("NumberOfProcessedInputs <= NumberOfInputs && \"Processing more inputs that actually exist!\"", "/home/seafit/code_projects/clang_source/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp", 497, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true"> "Processing more inputs that actually exist!");
498	(0) . __assert_fail ("HostInputIndex != ~0u && \"Host input index not defined.\"", "/home/seafit/code_projects/clang_source/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp", 498, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(HostInputIndex != ~0u && "Host input index not defined.");
499
500	// If this is not the last output, we don't have to do anything.
501	if (NumberOfProcessedInputs != NumberOfInputs)
502	return false;
503
504	// Create the bitcode file name to write the resulting code to. Keep it if
505	// save-temps is active.
506	SmallString<128> BitcodeFileName;
507	if (sys::fs::createTemporaryFile("clang-offload-bundler", "bc",
508	BitcodeFileName)) {
509	errs() << "error: unable to create temporary file.\n";
510	return true;
511	}
512
513	// Dump the contents of the temporary file if that was requested.
514	if (DumpTemporaryFiles) {
515	errs() << ";\n; Object file bundler IR file.\n;\n";
516	AuxModule.get()->print(errs(), nullptr,
517	/ShouldPreserveUseListOrder=/false,
518	/IsForDebug=/true);
519	errs() << '\n';
520	}
521
522	// Find clang in order to create the bundle binary.
523	StringRef Dir = sys::path::parent_path(BundlerExecutable);
524
525	auto ClangBinary = sys::findProgramByName("clang", Dir);
526	if (ClangBinary.getError()) {
527	// Remove bitcode file.
528	sys::fs::remove(BitcodeFileName);
529
530	errs() << "error: unable to find 'clang' in path.\n";
531	return true;
532	}
533
534	// Do the incremental linking. We write to the output file directly. So, we
535	// close it and use the name to pass down to clang.
536	OS.close();
537	SmallString<128> TargetName = getTriple(TargetNames[HostInputIndex]);
538	std::vector<StringRef> ClangArgs = {"clang",
539	"-r",
540	"-target",
541	TargetName.c_str(),
542	"-o",
543	OutputFileNames.front().c_str(),
544	InputFileNames[HostInputIndex].c_str(),
545	BitcodeFileName.c_str(),
546	"-nostdlib"};
547
548	// If the user asked for the commands to be printed out, we do that instead
549	// of executing it.
550	if (PrintExternalCommands) {
551	errs() << "\"" << ClangBinary.get() << "\"";
552	for (StringRef Arg : ClangArgs)
553	errs() << " \"" << Arg << "\"";
554	errs() << "\n";
555	} else {
556	// Write the bitcode contents to the temporary file.
557	{
558	std::error_code EC;
559	raw_fd_ostream BitcodeFile(BitcodeFileName, EC, sys::fs::F_None);
560	if (EC) {
561	errs() << "error: unable to open temporary file.\n";
562	return true;
563	}
564	WriteBitcodeToFile(*AuxModule, BitcodeFile);
565	}
566
567	bool Failed = sys::ExecuteAndWait(ClangBinary.get(), ClangArgs);
568
569	// Remove bitcode file.
570	sys::fs::remove(BitcodeFileName);
571
572	if (Failed) {
573	errs() << "error: incremental linking by external tool failed.\n";
574	return true;
575	}
576	}
577
578	return false;
579	}
580
581	void WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
582	Module *M = AuxModule.get();
583
584	// Create the new section name, it will consist of the reserved prefix
585	// concatenated with the triple.
586	std::string SectionName = OFFLOAD_BUNDLER_MAGIC_STR;
587	SectionName += CurrentTriple;
588
589	// Create the constant with the content of the section. For the input we are
590	// bundling into (the host input), this is just a place-holder, so a single
591	// byte is sufficient.
592	(0) . __assert_fail ("HostInputIndex != ~0u && \"Host input index undefined??\"", "/home/seafit/code_projects/clang_source/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp", 592, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(HostInputIndex != ~0u && "Host input index undefined??");
593	Constant *Content;
594	if (NumberOfProcessedInputs == HostInputIndex + 1) {
595	uint8_t Byte[] = {0};
596	Content = ConstantDataArray::get(VMContext, Byte);
597	} else
598	Content = ConstantDataArray::get(
599	VMContext, ArrayRef<uint8_t>(reinterpret_cast<const uint8_t *>(
600	Input.getBufferStart()),
601	Input.getBufferSize()));
602
603	// Create the global in the desired section. We don't want these globals in
604	// the symbol table, so we mark them private.
605	auto GV = new GlobalVariable(M, Content->getType(), /IsConstant=/true,
606	GlobalVariable::PrivateLinkage, Content);
607	GV->setSection(SectionName);
608	}
609	};
610
611	/// Handler for text files. The bundled file will have the following format.
612	///
613	/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple"
614	/// Bundle 1
615	/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple"
616	/// ...
617	/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple"
618	/// Bundle N
619	/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple"
620	class TextFileHandler final : public FileHandler {
621	/// String that begins a line comment.
622	StringRef Comment;
623
624	/// String that initiates a bundle.
625	std::string BundleStartString;
626
627	/// String that closes a bundle.
628	std::string BundleEndString;
629
630	/// Number of chars read from input.
631	size_t ReadChars = 0u;
632
633	protected:
634	void ReadHeader(MemoryBuffer &Input) final {}
635
636	StringRef ReadBundleStart(MemoryBuffer &Input) final {
637	StringRef FC = Input.getBuffer();
638
639	// Find start of the bundle.
640	ReadChars = FC.find(BundleStartString, ReadChars);
641	if (ReadChars == FC.npos)
642	return StringRef();
643
644	// Get position of the triple.
645	size_t TripleStart = ReadChars = ReadChars + BundleStartString.size();
646
647	// Get position that closes the triple.
648	size_t TripleEnd = ReadChars = FC.find("\n", ReadChars);
649	if (TripleEnd == FC.npos)
650	return StringRef();
651
652	// Next time we read after the new line.
653	++ReadChars;
654
655	return StringRef(&FC.data()[TripleStart], TripleEnd - TripleStart);
656	}
657
658	void ReadBundleEnd(MemoryBuffer &Input) final {
659	StringRef FC = Input.getBuffer();
660
661	// Read up to the next new line.
662	(0) . __assert_fail ("FC[ReadChars] == '\\n' && \"The bundle should end with a new line.\"", "/home/seafit/code_projects/clang_source/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp", 662, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(FC[ReadChars] == '\n' && "The bundle should end with a new line.");
663
664	size_t TripleEnd = ReadChars = FC.find("\n", ReadChars + 1);
665	if (TripleEnd == FC.npos)
666	return;
667
668	// Next time we read after the new line.
669	++ReadChars;
670	}
671
672	void ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
673	StringRef FC = Input.getBuffer();
674	size_t BundleStart = ReadChars;
675
676	// Find end of the bundle.
677	size_t BundleEnd = ReadChars = FC.find(BundleEndString, ReadChars);
678
679	StringRef Bundle(&FC.data()[BundleStart], BundleEnd - BundleStart);
680	OS << Bundle;
681	}
682
683	void WriteHeader(raw_fd_ostream &OS,
684	ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) final {}
685
686	void WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) final {
687	OS << BundleStartString << TargetTriple << "\n";
688	}
689
690	bool WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) final {
691	OS << BundleEndString << TargetTriple << "\n";
692	return false;
693	}
694
695	void WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
696	OS << Input.getBuffer();
697	}
698
699	public:
700	TextFileHandler(StringRef Comment)
701	: FileHandler(), Comment(Comment), ReadChars(0) {
702	BundleStartString =
703	"\n" + Comment.str() + " " OFFLOAD_BUNDLER_MAGIC_STR "__START__ ";
704	BundleEndString =
705	"\n" + Comment.str() + " " OFFLOAD_BUNDLER_MAGIC_STR "__END__ ";
706	}
707	};
708
709	/// Return an appropriate object file handler. We use the specific object
710	/// handler if we know how to deal with that format, otherwise we use a default
711	/// binary file handler.
712	static FileHandler *CreateObjectFileHandler(MemoryBuffer &FirstInput) {
713	// Check if the input file format is one that we know how to deal with.
714	Expected<std::unique_ptr<Binary>> BinaryOrErr = createBinary(FirstInput);
715
716	// Failed to open the input as a known binary. Use the default binary handler.
717	if (!BinaryOrErr) {
718	// We don't really care about the error (we just consume it), if we could
719	// not get a valid device binary object we use the default binary handler.
720	consumeError(BinaryOrErr.takeError());
721	return new BinaryFileHandler();
722	}
723
724	// We only support regular object files. If this is not an object file,
725	// default to the binary handler. The handler will be owned by the client of
726	// this function.
727	std::unique_ptr<ObjectFile> Obj(
728	dyn_cast<ObjectFile>(BinaryOrErr.get().release()));
729
730	if (!Obj)
731	return new BinaryFileHandler();
732
733	return new ObjectFileHandler(std::move(Obj));
734	}
735
736	/// Return an appropriate handler given the input files and options.
737	static FileHandler *CreateFileHandler(MemoryBuffer &FirstInput) {
738	if (FilesType == "i")
739	return new TextFileHandler(/Comment=/"//");
740	if (FilesType == "ii")
741	return new TextFileHandler(/Comment=/"//");
742	if (FilesType == "ll")
743	return new TextFileHandler(/Comment=/";");
744	if (FilesType == "bc")
745	return new BinaryFileHandler();
746	if (FilesType == "s")
747	return new TextFileHandler(/Comment=/"#");
748	if (FilesType == "o")
749	return CreateObjectFileHandler(FirstInput);
750	if (FilesType == "gch")
751	return new BinaryFileHandler();
752	if (FilesType == "ast")
753	return new BinaryFileHandler();
754
755	errs() << "error: invalid file type specified.\n";
756	return nullptr;
757	}
758
759	/// Bundle the files. Return true if an error was found.
760	static bool BundleFiles() {
761	std::error_code EC;
762
763	// Create output file.
764	raw_fd_ostream OutputFile(OutputFileNames.front(), EC, sys::fs::F_None);
765
766	if (EC) {
767	errs() << "error: Can't open file " << OutputFileNames.front() << ".\n";
768	return true;
769	}
770
771	// Open input files.
772	std::vector<std::unique_ptr<MemoryBuffer>> InputBuffers(
773	InputFileNames.size());
774
775	unsigned Idx = 0;
776	for (auto &I : InputFileNames) {
777	ErrorOr<std::unique_ptr<MemoryBuffer>> CodeOrErr =
778	MemoryBuffer::getFileOrSTDIN(I);
779	if (std::error_code EC = CodeOrErr.getError()) {
780	errs() << "error: Can't open file " << I << ": " << EC.message() << "\n";
781	return true;
782	}
783	InputBuffers[Idx++] = std::move(CodeOrErr.get());
784	}
785
786	// Get the file handler. We use the host buffer as reference.
787	(0) . __assert_fail ("HostInputIndex != ~0u && \"Host input index undefined??\"", "/home/seafit/code_projects/clang_source/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp", 787, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(HostInputIndex != ~0u && "Host input index undefined??");
788	std::unique_ptr<FileHandler> FH;
789	FH.reset(CreateFileHandler(*InputBuffers[HostInputIndex].get()));
790
791	// Quit if we don't have a handler.
792	if (!FH.get())
793	return true;
794
795	// Write header.
796	FH.get()->WriteHeader(OutputFile, InputBuffers);
797
798	// Write all bundles along with the start/end markers. If an error was found
799	// writing the end of the bundle component, abort the bundle writing.
800	auto Input = InputBuffers.begin();
801	for (auto &Triple : TargetNames) {
802	FH.get()->WriteBundleStart(OutputFile, Triple);
803	FH.get()->WriteBundle(OutputFile, *Input->get());
804	if (FH.get()->WriteBundleEnd(OutputFile, Triple))
805	return true;
806	++Input;
807	}
808	return false;
809	}
810
811	// Unbundle the files. Return true if an error was found.
812	static bool UnbundleFiles() {
813	// Open Input file.
814	ErrorOr<std::unique_ptr<MemoryBuffer>> CodeOrErr =
815	MemoryBuffer::getFileOrSTDIN(InputFileNames.front());
816	if (std::error_code EC = CodeOrErr.getError()) {
817	errs() << "error: Can't open file " << InputFileNames.front() << ": "
818	<< EC.message() << "\n";
819	return true;
820	}
821
822	MemoryBuffer &Input = *CodeOrErr.get();
823
824	// Select the right files handler.
825	std::unique_ptr<FileHandler> FH;
826	FH.reset(CreateFileHandler(Input));
827
828	// Quit if we don't have a handler.
829	if (!FH.get())
830	return true;
831
832	// Read the header of the bundled file.
833	FH.get()->ReadHeader(Input);
834
835	// Create a work list that consist of the map triple/output file.
836	StringMap<StringRef> Worklist;
837	auto Output = OutputFileNames.begin();
838	for (auto &Triple : TargetNames) {
839	Worklist[Triple] = *Output;
840	++Output;
841	}
842
843	// Read all the bundles that are in the work list. If we find no bundles we
844	// assume the file is meant for the host target.
845	bool FoundHostBundle = false;
846	while (!Worklist.empty()) {
847	StringRef CurTriple = FH.get()->ReadBundleStart(Input);
848
849	// We don't have more bundles.
850	if (CurTriple.empty())
851	break;
852
853	auto Output = Worklist.find(CurTriple);
854	// The file may have more bundles for other targets, that we don't care
855	// about. Therefore, move on to the next triple
856	if (Output == Worklist.end()) {
857	continue;
858	}
859
860	// Check if the output file can be opened and copy the bundle to it.
861	std::error_code EC;
862	raw_fd_ostream OutputFile(Output->second, EC, sys::fs::F_None);
863	if (EC) {
864	errs() << "error: Can't open file " << Output->second << ": "
865	<< EC.message() << "\n";
866	return true;
867	}
868	FH.get()->ReadBundle(OutputFile, Input);
869	FH.get()->ReadBundleEnd(Input);
870	Worklist.erase(Output);
871
872	// Record if we found the host bundle.
873	if (hasHostKind(CurTriple))
874	FoundHostBundle = true;
875	}
876
877	// If no bundles were found, assume the input file is the host bundle and
878	// create empty files for the remaining targets.
879	if (Worklist.size() == TargetNames.size()) {
880	for (auto &E : Worklist) {
881	std::error_code EC;
882	raw_fd_ostream OutputFile(E.second, EC, sys::fs::F_None);
883	if (EC) {
884	errs() << "error: Can't open file " << E.second << ": " << EC.message()
885	<< "\n";
886	return true;
887	}
888
889	// If this entry has a host kind, copy the input file to the output file.
890	if (hasHostKind(E.first()))
891	OutputFile.write(Input.getBufferStart(), Input.getBufferSize());
892	}
893	return false;
894	}
895
896	// If we found elements, we emit an error if none of those were for the host.
897	if (!FoundHostBundle) {
898	errs() << "error: Can't find bundle for the host target\n";
899	return true;
900	}
901
902	// If we still have any elements in the worklist, create empty files for them.
903	for (auto &E : Worklist) {
904	std::error_code EC;
905	raw_fd_ostream OutputFile(E.second, EC, sys::fs::F_None);
906	if (EC) {
907	errs() << "error: Can't open file " << E.second << ": " << EC.message()
908	<< "\n";
909	return true;
910	}
911	}
912
913	return false;
914	}
915
916	static void PrintVersion(raw_ostream &OS) {
917	OS << clang::getClangToolFullVersion("clang-offload-bundler") << '\n';
918	}
919
920	int main(int argc, const char **argv) {
921	sys::PrintStackTraceOnErrorSignal(argv[0]);
922
923	cl::HideUnrelatedOptions(ClangOffloadBundlerCategory);
924	cl::SetVersionPrinter(PrintVersion);
925	cl::ParseCommandLineOptions(
926	argc, argv,
927	"A tool to bundle several input files of the specified type <type> \n"
928	"referring to the same source file but different targets into a single \n"
929	"one. The resulting file can also be unbundled into different files by \n"
930	"this tool if -unbundle is provided.\n");
931
932	if (Help) {
933	cl::PrintHelpMessage();
934	return 0;
935	}
936
937	bool Error = false;
938	if (Unbundle) {
939	if (InputFileNames.size() != 1) {
940	Error = true;
941	errs() << "error: only one input file supported in unbundling mode.\n";
942	}
943	if (OutputFileNames.size() != TargetNames.size()) {
944	Error = true;
945	errs() << "error: number of output files and targets should match in "
946	"unbundling mode.\n";
947	}
948	} else {
949	if (OutputFileNames.size() != 1) {
950	Error = true;
951	errs() << "error: only one output file supported in bundling mode.\n";
952	}
953	if (InputFileNames.size() != TargetNames.size()) {
954	Error = true;
955	errs() << "error: number of input files and targets should match in "
956	"bundling mode.\n";
957	}
958	}
959
960	// Verify that the offload kinds and triples are known. We also check that we
961	// have exactly one host target.
962	unsigned Index = 0u;
963	unsigned HostTargetNum = 0u;
964	for (StringRef Target : TargetNames) {
965	StringRef Kind;
966	StringRef Triple;
967	getOffloadKindAndTriple(Target, Kind, Triple);
968
969	bool KindIsValid = !Kind.empty();
970	KindIsValid = KindIsValid && StringSwitch<bool>(Kind)
971	.Case("host", true)
972	.Case("openmp", true)
973	.Case("hip", true)
974	.Default(false);
975
976	bool TripleIsValid = !Triple.empty();
977	llvm::Triple T(Triple);
978	TripleIsValid &= T.getArch() != Triple::UnknownArch;
979
980	if (!KindIsValid \|\| !TripleIsValid) {
981	Error = true;
982	errs() << "error: invalid target '" << Target << "'";
983
984	if (!KindIsValid)
985	errs() << ", unknown offloading kind '" << Kind << "'";
986	if (!TripleIsValid)
987	errs() << ", unknown target triple '" << Triple << "'";
988	errs() << ".\n";
989	}
990
991	if (KindIsValid && Kind == "host") {
992	++HostTargetNum;
993	// Save the index of the input that refers to the host.
994	HostInputIndex = Index;
995	}
996
997	++Index;
998	}
999
1000	if (HostTargetNum != 1) {
1001	Error = true;
1002	errs() << "error: expecting exactly one host target but got "
1003	<< HostTargetNum << ".\n";
1004	}
1005
1006	if (Error)
1007	return 1;
1008
1009	// Save the current executable directory as it will be useful to find other
1010	// tools.
1011	BundlerExecutable = sys::fs::getMainExecutable(argv[0], &BundlerExecutable);
1012
1013	return Unbundle ? UnbundleFiles() : BundleFiles();
1014	}
1015

FileHandler::ReadHeader

FileHandler::ReadBundleEnd

FileHandler::ReadBundle

FileHandler::WriteHeader

FileHandler::WriteBundleStart

FileHandler::WriteBundleEnd

FileHandler::WriteBundle

Clang Project