Update prebuilt Clang to r416183b from Android.
https://android.googlesource.com/platform/prebuilts/clang/host/
linux-x86/+/06a71ddac05c22edb2d10b590e1769b3f8619bef
clang 12.0.5 (based on r416183b) from build 7284624.
Change-Id: I277a316abcf47307562d8b748b84870f31a72866
Signed-off-by: Olivier Deprez <olivier.deprez@arm.com>
diff --git a/linux-x64/clang/include/llvm/ProfileData/Coverage/CoverageMapping.h b/linux-x64/clang/include/llvm/ProfileData/Coverage/CoverageMapping.h
index 11758ac..09f2167 100644
--- a/linux-x64/clang/include/llvm/ProfileData/Coverage/CoverageMapping.h
+++ b/linux-x64/clang/include/llvm/ProfileData/Coverage/CoverageMapping.h
@@ -20,10 +20,10 @@
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/StringSet.h"
#include "llvm/ADT/iterator.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/Support/Alignment.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/Endian.h"
@@ -54,7 +54,9 @@
no_data_found,
unsupported_version,
truncated,
- malformed
+ malformed,
+ decompression_failed,
+ invalid_or_missing_arch_specifier
};
const std::error_category &coveragemap_category();
@@ -88,6 +90,8 @@
/// A Counter is an abstract value that describes how to compute the
/// execution count for a region of code using the collected profile count data.
struct Counter {
+ /// The CounterExpression kind (Add or Subtract) is encoded in bit 0 next to
+ /// the CounterKind. This means CounterKind has to leave bit 0 free.
enum CounterKind { Zero, CounterValueReference, Expression };
static const unsigned EncodingTagBits = 2;
static const unsigned EncodingTagMask = 0x3;
@@ -217,10 +221,20 @@
/// A GapRegion is like a CodeRegion, but its count is only set as the
/// line execution count when its the only region in the line.
- GapRegion
+ GapRegion,
+
+ /// A BranchRegion represents leaf-level boolean expressions and is
+ /// associated with two counters, each representing the number of times the
+ /// expression evaluates to true or false.
+ BranchRegion
};
+ /// Primary Counter that is also used for Branch Regions (TrueCount).
Counter Count;
+
+ /// Secondary Counter used for Branch Regions (FalseCount).
+ Counter FalseCount;
+
unsigned FileID, ExpandedFileID;
unsigned LineStart, ColumnStart, LineEnd, ColumnEnd;
RegionKind Kind;
@@ -232,6 +246,15 @@
LineStart(LineStart), ColumnStart(ColumnStart), LineEnd(LineEnd),
ColumnEnd(ColumnEnd), Kind(Kind) {}
+ CounterMappingRegion(Counter Count, Counter FalseCount, unsigned FileID,
+ unsigned ExpandedFileID, unsigned LineStart,
+ unsigned ColumnStart, unsigned LineEnd,
+ unsigned ColumnEnd, RegionKind Kind)
+ : Count(Count), FalseCount(FalseCount), FileID(FileID),
+ ExpandedFileID(ExpandedFileID), LineStart(LineStart),
+ ColumnStart(ColumnStart), LineEnd(LineEnd), ColumnEnd(ColumnEnd),
+ Kind(Kind) {}
+
static CounterMappingRegion
makeRegion(Counter Count, unsigned FileID, unsigned LineStart,
unsigned ColumnStart, unsigned LineEnd, unsigned ColumnEnd) {
@@ -261,6 +284,14 @@
LineEnd, (1U << 31) | ColumnEnd, GapRegion);
}
+ static CounterMappingRegion
+ makeBranchRegion(Counter Count, Counter FalseCount, unsigned FileID,
+ unsigned LineStart, unsigned ColumnStart, unsigned LineEnd,
+ unsigned ColumnEnd) {
+ return CounterMappingRegion(Count, FalseCount, FileID, 0, LineStart,
+ ColumnStart, LineEnd, ColumnEnd, BranchRegion);
+ }
+
inline LineColPair startLoc() const {
return LineColPair(LineStart, ColumnStart);
}
@@ -271,9 +302,17 @@
/// Associates a source range with an execution count.
struct CountedRegion : public CounterMappingRegion {
uint64_t ExecutionCount;
+ uint64_t FalseExecutionCount;
+ bool Folded;
CountedRegion(const CounterMappingRegion &R, uint64_t ExecutionCount)
- : CounterMappingRegion(R), ExecutionCount(ExecutionCount) {}
+ : CounterMappingRegion(R), ExecutionCount(ExecutionCount),
+ FalseExecutionCount(0), Folded(false) {}
+
+ CountedRegion(const CounterMappingRegion &R, uint64_t ExecutionCount,
+ uint64_t FalseExecutionCount)
+ : CounterMappingRegion(R), ExecutionCount(ExecutionCount),
+ FalseExecutionCount(FalseExecutionCount), Folded(false) {}
};
/// A Counter mapping context is used to connect the counters, expressions
@@ -301,12 +340,19 @@
struct FunctionRecord {
/// Raw function name.
std::string Name;
- /// Associated files.
+ /// Mapping from FileID (i.e. vector index) to filename. Used to support
+ /// macro expansions within a function in which the macro and function are
+ /// defined in separate files.
+ ///
+ /// TODO: Uniquing filenames across all function records may be a performance
+ /// optimization.
std::vector<std::string> Filenames;
/// Regions in the function along with their counts.
std::vector<CountedRegion> CountedRegions;
+ /// Branch Regions in the function along with their counts.
+ std::vector<CountedRegion> CountedBranchRegions;
/// The number of times this function was executed.
- uint64_t ExecutionCount;
+ uint64_t ExecutionCount = 0;
FunctionRecord(StringRef Name, ArrayRef<StringRef> Filenames)
: Name(Name), Filenames(Filenames.begin(), Filenames.end()) {}
@@ -314,10 +360,19 @@
FunctionRecord(FunctionRecord &&FR) = default;
FunctionRecord &operator=(FunctionRecord &&) = default;
- void pushRegion(CounterMappingRegion Region, uint64_t Count) {
+ void pushRegion(CounterMappingRegion Region, uint64_t Count,
+ uint64_t FalseCount) {
+ if (Region.Kind == CounterMappingRegion::BranchRegion) {
+ CountedBranchRegions.emplace_back(Region, Count, FalseCount);
+ // If both counters are hard-coded to zero, then this region represents a
+ // constant-folded branch.
+ if (Region.Count.isZero() && Region.FalseCount.isZero())
+ CountedBranchRegions.back().Folded = true;
+ return;
+ }
if (CountedRegions.empty())
ExecutionCount = Count;
- CountedRegions.emplace_back(Region, Count);
+ CountedRegions.emplace_back(Region, Count, FalseCount);
}
};
@@ -396,7 +451,8 @@
IsRegionEntry(IsRegionEntry), IsGapRegion(false) {}
CoverageSegment(unsigned Line, unsigned Col, uint64_t Count,
- bool IsRegionEntry, bool IsGapRegion = false)
+ bool IsRegionEntry, bool IsGapRegion = false,
+ bool IsBranchRegion = false)
: Line(Line), Col(Col), Count(Count), HasCount(true),
IsRegionEntry(IsRegionEntry), IsGapRegion(IsGapRegion) {}
@@ -476,6 +532,7 @@
std::string Filename;
std::vector<CoverageSegment> Segments;
std::vector<ExpansionRecord> Expansions;
+ std::vector<CountedRegion> BranchRegions;
public:
CoverageData() = default;
@@ -499,6 +556,9 @@
/// Expansions that can be further processed.
ArrayRef<ExpansionRecord> getExpansions() const { return Expansions; }
+
+ /// Branches that can be further processed.
+ ArrayRef<CountedRegion> getBranches() const { return BranchRegions; }
};
/// The mapping of profile information to coverage data.
@@ -508,6 +568,7 @@
class CoverageMapping {
DenseMap<size_t, DenseSet<size_t>> RecordProvenance;
std::vector<FunctionRecord> Functions;
+ DenseMap<size_t, SmallVector<unsigned, 0>> FilenameHash2RecordIndices;
std::vector<std::pair<std::string, uint64_t>> FuncHashMismatches;
CoverageMapping() = default;
@@ -516,6 +577,13 @@
Error loadFunctionRecord(const CoverageMappingRecord &Record,
IndexedInstrProfReader &ProfileReader);
+ /// Look up the indices for function records which are at least partially
+ /// defined in the specified file. This is guaranteed to return a superset of
+ /// such records: extra records not in the file may be included if there is
+ /// a hash collision on the filename. Clients must be robust to collisions.
+ ArrayRef<unsigned>
+ getImpreciseRecordIndicesForFilename(StringRef Filename) const;
+
public:
CoverageMapping(const CoverageMapping &) = delete;
CoverageMapping &operator=(const CoverageMapping &) = delete;
@@ -527,6 +595,7 @@
/// Load the coverage mapping from the given object files and profile. If
/// \p Arches is non-empty, it must specify an architecture for each object.
+ /// Ignores non-instrumented object files unless all are not instrumented.
static Expected<std::unique_ptr<CoverageMapping>>
load(ArrayRef<StringRef> ObjectFilenames, StringRef ProfileFilename,
ArrayRef<StringRef> Arches = None);
@@ -664,37 +733,107 @@
return make_range(Begin, End);
}
-// Profile coverage map has the following layout:
-// [CoverageMapFileHeader]
-// [ArrayStart]
-// [CovMapFunctionRecord]
-// [CovMapFunctionRecord]
-// ...
-// [ArrayEnd]
-// [Encoded Region Mapping Data]
+// Coverage mappping data (V2) has the following layout:
+// IPSK_covmap:
+// [CoverageMapFileHeader]
+// [ArrayStart]
+// [CovMapFunctionRecordV2]
+// [CovMapFunctionRecordV2]
+// ...
+// [ArrayEnd]
+// [Encoded Filenames and Region Mapping Data]
+//
+// Coverage mappping data (V3) has the following layout:
+// IPSK_covmap:
+// [CoverageMapFileHeader]
+// [Encoded Filenames]
+// IPSK_covfun:
+// [ArrayStart]
+// odr_name_1: [CovMapFunctionRecordV3]
+// odr_name_2: [CovMapFunctionRecordV3]
+// ...
+// [ArrayEnd]
+//
+// Both versions of the coverage mapping format encode the same information,
+// but the V3 format does so more compactly by taking advantage of linkonce_odr
+// semantics (it allows exactly 1 function record per name reference).
+
+/// This namespace defines accessors shared by different versions of coverage
+/// mapping records.
+namespace accessors {
+
+/// Return the structural hash associated with the function.
+template <class FuncRecordTy, support::endianness Endian>
+uint64_t getFuncHash(const FuncRecordTy *Record) {
+ return support::endian::byte_swap<uint64_t, Endian>(Record->FuncHash);
+}
+
+/// Return the coverage map data size for the function.
+template <class FuncRecordTy, support::endianness Endian>
+uint64_t getDataSize(const FuncRecordTy *Record) {
+ return support::endian::byte_swap<uint32_t, Endian>(Record->DataSize);
+}
+
+/// Return the function lookup key. The value is considered opaque.
+template <class FuncRecordTy, support::endianness Endian>
+uint64_t getFuncNameRef(const FuncRecordTy *Record) {
+ return support::endian::byte_swap<uint64_t, Endian>(Record->NameRef);
+}
+
+/// Return the PGO name of the function. Used for formats in which the name is
+/// a hash.
+template <class FuncRecordTy, support::endianness Endian>
+Error getFuncNameViaRef(const FuncRecordTy *Record,
+ InstrProfSymtab &ProfileNames, StringRef &FuncName) {
+ uint64_t NameRef = getFuncNameRef<FuncRecordTy, Endian>(Record);
+ FuncName = ProfileNames.getFuncName(NameRef);
+ return Error::success();
+}
+
+/// Read coverage mapping out-of-line, from \p MappingBuf. This is used when the
+/// coverage mapping is attached to the file header, instead of to the function
+/// record.
+template <class FuncRecordTy, support::endianness Endian>
+StringRef getCoverageMappingOutOfLine(const FuncRecordTy *Record,
+ const char *MappingBuf) {
+ return {MappingBuf, size_t(getDataSize<FuncRecordTy, Endian>(Record))};
+}
+
+/// Advance to the next out-of-line coverage mapping and its associated
+/// function record.
+template <class FuncRecordTy, support::endianness Endian>
+std::pair<const char *, const FuncRecordTy *>
+advanceByOneOutOfLine(const FuncRecordTy *Record, const char *MappingBuf) {
+ return {MappingBuf + getDataSize<FuncRecordTy, Endian>(Record), Record + 1};
+}
+
+} // end namespace accessors
+
LLVM_PACKED_START
-template <class IntPtrT> struct CovMapFunctionRecordV1 {
+template <class IntPtrT>
+struct CovMapFunctionRecordV1 {
+ using ThisT = CovMapFunctionRecordV1<IntPtrT>;
+
#define COVMAP_V1
#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init) Type Name;
#include "llvm/ProfileData/InstrProfData.inc"
#undef COVMAP_V1
+ CovMapFunctionRecordV1() = delete;
- // Return the structural hash associated with the function.
template <support::endianness Endian> uint64_t getFuncHash() const {
- return support::endian::byte_swap<uint64_t, Endian>(FuncHash);
+ return accessors::getFuncHash<ThisT, Endian>(this);
}
- // Return the coverage map data size for the funciton.
- template <support::endianness Endian> uint32_t getDataSize() const {
- return support::endian::byte_swap<uint32_t, Endian>(DataSize);
+ template <support::endianness Endian> uint64_t getDataSize() const {
+ return accessors::getDataSize<ThisT, Endian>(this);
}
- // Return function lookup key. The value is consider opaque.
+ /// Return function lookup key. The value is consider opaque.
template <support::endianness Endian> IntPtrT getFuncNameRef() const {
return support::endian::byte_swap<IntPtrT, Endian>(NamePtr);
}
- // Return the PGO name of the function */
+ /// Return the PGO name of the function.
template <support::endianness Endian>
Error getFuncName(InstrProfSymtab &ProfileNames, StringRef &FuncName) const {
IntPtrT NameRef = getFuncNameRef<Endian>();
@@ -704,33 +843,119 @@
return make_error<CoverageMapError>(coveragemap_error::malformed);
return Error::success();
}
+
+ template <support::endianness Endian>
+ std::pair<const char *, const ThisT *>
+ advanceByOne(const char *MappingBuf) const {
+ return accessors::advanceByOneOutOfLine<ThisT, Endian>(this, MappingBuf);
+ }
+
+ template <support::endianness Endian> uint64_t getFilenamesRef() const {
+ llvm_unreachable("V1 function format does not contain a filenames ref");
+ }
+
+ template <support::endianness Endian>
+ StringRef getCoverageMapping(const char *MappingBuf) const {
+ return accessors::getCoverageMappingOutOfLine<ThisT, Endian>(this,
+ MappingBuf);
+ }
};
-struct CovMapFunctionRecord {
+struct CovMapFunctionRecordV2 {
+ using ThisT = CovMapFunctionRecordV2;
+
+#define COVMAP_V2
#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init) Type Name;
#include "llvm/ProfileData/InstrProfData.inc"
+#undef COVMAP_V2
+ CovMapFunctionRecordV2() = delete;
- // Return the structural hash associated with the function.
template <support::endianness Endian> uint64_t getFuncHash() const {
- return support::endian::byte_swap<uint64_t, Endian>(FuncHash);
+ return accessors::getFuncHash<ThisT, Endian>(this);
}
- // Return the coverage map data size for the funciton.
- template <support::endianness Endian> uint32_t getDataSize() const {
- return support::endian::byte_swap<uint32_t, Endian>(DataSize);
+ template <support::endianness Endian> uint64_t getDataSize() const {
+ return accessors::getDataSize<ThisT, Endian>(this);
}
- // Return function lookup key. The value is consider opaque.
template <support::endianness Endian> uint64_t getFuncNameRef() const {
- return support::endian::byte_swap<uint64_t, Endian>(NameRef);
+ return accessors::getFuncNameRef<ThisT, Endian>(this);
}
- // Return the PGO name of the function */
template <support::endianness Endian>
Error getFuncName(InstrProfSymtab &ProfileNames, StringRef &FuncName) const {
- uint64_t NameRef = getFuncNameRef<Endian>();
- FuncName = ProfileNames.getFuncName(NameRef);
- return Error::success();
+ return accessors::getFuncNameViaRef<ThisT, Endian>(this, ProfileNames,
+ FuncName);
+ }
+
+ template <support::endianness Endian>
+ std::pair<const char *, const ThisT *>
+ advanceByOne(const char *MappingBuf) const {
+ return accessors::advanceByOneOutOfLine<ThisT, Endian>(this, MappingBuf);
+ }
+
+ template <support::endianness Endian> uint64_t getFilenamesRef() const {
+ llvm_unreachable("V2 function format does not contain a filenames ref");
+ }
+
+ template <support::endianness Endian>
+ StringRef getCoverageMapping(const char *MappingBuf) const {
+ return accessors::getCoverageMappingOutOfLine<ThisT, Endian>(this,
+ MappingBuf);
+ }
+};
+
+struct CovMapFunctionRecordV3 {
+ using ThisT = CovMapFunctionRecordV3;
+
+#define COVMAP_V3
+#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init) Type Name;
+#include "llvm/ProfileData/InstrProfData.inc"
+#undef COVMAP_V3
+ CovMapFunctionRecordV3() = delete;
+
+ template <support::endianness Endian> uint64_t getFuncHash() const {
+ return accessors::getFuncHash<ThisT, Endian>(this);
+ }
+
+ template <support::endianness Endian> uint64_t getDataSize() const {
+ return accessors::getDataSize<ThisT, Endian>(this);
+ }
+
+ template <support::endianness Endian> uint64_t getFuncNameRef() const {
+ return accessors::getFuncNameRef<ThisT, Endian>(this);
+ }
+
+ template <support::endianness Endian>
+ Error getFuncName(InstrProfSymtab &ProfileNames, StringRef &FuncName) const {
+ return accessors::getFuncNameViaRef<ThisT, Endian>(this, ProfileNames,
+ FuncName);
+ }
+
+ /// Get the filename set reference.
+ template <support::endianness Endian> uint64_t getFilenamesRef() const {
+ return support::endian::byte_swap<uint64_t, Endian>(FilenamesRef);
+ }
+
+ /// Read the inline coverage mapping. Ignore the buffer parameter, it is for
+ /// out-of-line coverage mapping data only.
+ template <support::endianness Endian>
+ StringRef getCoverageMapping(const char *) const {
+ return StringRef(&CoverageMapping, getDataSize<Endian>());
+ }
+
+ // Advance to the next inline coverage mapping and its associated function
+ // record. Ignore the out-of-line coverage mapping buffer.
+ template <support::endianness Endian>
+ std::pair<const char *, const CovMapFunctionRecordV3 *>
+ advanceByOne(const char *) const {
+ assert(isAddrAligned(Align(8), this) && "Function record not aligned");
+ const char *Next = ((const char *)this) + sizeof(CovMapFunctionRecordV3) -
+ sizeof(char) + getDataSize<Endian>();
+ // Each function record has an alignment of 8, so we need to adjust
+ // alignment before reading the next record.
+ Next += offsetToAlignedAddr(Next, Align(8));
+ return {nullptr, reinterpret_cast<const CovMapFunctionRecordV3 *>(Next)};
}
};
@@ -767,12 +992,26 @@
// A new interpretation of the columnEnd field is added in order to mark
// regions as gap areas.
Version3 = 2,
- // The current version is Version3
+ // Function records are named, uniqued, and moved to a dedicated section.
+ Version4 = 3,
+ // Branch regions referring to two counters are added
+ Version5 = 4,
+ // The current version is Version5.
CurrentVersion = INSTR_PROF_COVMAP_VERSION
};
template <int CovMapVersion, class IntPtrT> struct CovMapTraits {
- using CovMapFuncRecordType = CovMapFunctionRecord;
+ using CovMapFuncRecordType = CovMapFunctionRecordV3;
+ using NameRefType = uint64_t;
+};
+
+template <class IntPtrT> struct CovMapTraits<CovMapVersion::Version3, IntPtrT> {
+ using CovMapFuncRecordType = CovMapFunctionRecordV2;
+ using NameRefType = uint64_t;
+};
+
+template <class IntPtrT> struct CovMapTraits<CovMapVersion::Version2, IntPtrT> {
+ using CovMapFuncRecordType = CovMapFunctionRecordV2;
using NameRefType = uint64_t;
};
diff --git a/linux-x64/clang/include/llvm/ProfileData/Coverage/CoverageMappingReader.h b/linux-x64/clang/include/llvm/ProfileData/Coverage/CoverageMappingReader.h
index 57a2aae..3a611bc 100644
--- a/linux-x64/clang/include/llvm/ProfileData/Coverage/CoverageMappingReader.h
+++ b/linux-x64/clang/include/llvm/ProfileData/Coverage/CoverageMappingReader.h
@@ -67,10 +67,10 @@
increment();
return *this;
}
- bool operator==(const CoverageMappingIterator &RHS) {
+ bool operator==(const CoverageMappingIterator &RHS) const {
return Reader == RHS.Reader;
}
- bool operator!=(const CoverageMappingIterator &RHS) {
+ bool operator!=(const CoverageMappingIterator &RHS) const {
return Reader != RHS.Reader;
}
Expected<CoverageMappingRecord &> operator*() {
@@ -113,20 +113,6 @@
Error readString(StringRef &Result);
};
-/// Reader for the raw coverage filenames.
-class RawCoverageFilenamesReader : public RawCoverageReader {
- std::vector<StringRef> &Filenames;
-
-public:
- RawCoverageFilenamesReader(StringRef Data, std::vector<StringRef> &Filenames)
- : RawCoverageReader(Data), Filenames(Filenames) {}
- RawCoverageFilenamesReader(const RawCoverageFilenamesReader &) = delete;
- RawCoverageFilenamesReader &
- operator=(const RawCoverageFilenamesReader &) = delete;
-
- Error read();
-};
-
/// Checks if the given coverage mapping data is exported for
/// an unused function.
class RawCoverageMappingDummyChecker : public RawCoverageReader {
@@ -188,6 +174,8 @@
FilenamesBegin(FilenamesBegin), FilenamesSize(FilenamesSize) {}
};
+ using DecompressedData = std::vector<std::unique_ptr<SmallVector<char, 0>>>;
+
private:
std::vector<StringRef> Filenames;
std::vector<ProfileMappingRecord> MappingRecords;
@@ -197,7 +185,17 @@
std::vector<CounterExpression> Expressions;
std::vector<CounterMappingRegion> MappingRegions;
- BinaryCoverageReader() = default;
+ // Used to tie the lifetimes of coverage function records to the lifetime of
+ // this BinaryCoverageReader instance. Needed to support the format change in
+ // D69471, which can split up function records into multiple sections on ELF.
+ std::string FuncRecords;
+
+ // Used to tie the lifetimes of decompressed strings to the lifetime of this
+ // BinaryCoverageReader instance.
+ DecompressedData Decompressed;
+
+ BinaryCoverageReader(std::string &&FuncRecords)
+ : FuncRecords(std::move(FuncRecords)) {}
public:
BinaryCoverageReader(const BinaryCoverageReader &) = delete;
@@ -208,7 +206,7 @@
SmallVectorImpl<std::unique_ptr<MemoryBuffer>> &ObjectFileBuffers);
static Expected<std::unique_ptr<BinaryCoverageReader>>
- createCoverageReaderFromBuffer(StringRef Coverage,
+ createCoverageReaderFromBuffer(StringRef Coverage, std::string &&FuncRecords,
InstrProfSymtab &&ProfileNames,
uint8_t BytesInAddress,
support::endianness Endian);
@@ -216,6 +214,24 @@
Error readNextRecord(CoverageMappingRecord &Record) override;
};
+/// Reader for the raw coverage filenames.
+class RawCoverageFilenamesReader : public RawCoverageReader {
+ std::vector<StringRef> &Filenames;
+
+ // Read an uncompressed sequence of filenames.
+ Error readUncompressed(uint64_t NumFilenames);
+
+public:
+ RawCoverageFilenamesReader(StringRef Data, std::vector<StringRef> &Filenames)
+ : RawCoverageReader(Data), Filenames(Filenames) {}
+ RawCoverageFilenamesReader(const RawCoverageFilenamesReader &) = delete;
+ RawCoverageFilenamesReader &
+ operator=(const RawCoverageFilenamesReader &) = delete;
+
+ Error read(CovMapVersion Version,
+ BinaryCoverageReader::DecompressedData &Decompressed);
+};
+
} // end namespace coverage
} // end namespace llvm
diff --git a/linux-x64/clang/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h b/linux-x64/clang/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h
index 5f88cac..303e518 100644
--- a/linux-x64/clang/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h
+++ b/linux-x64/clang/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h
@@ -30,11 +30,11 @@
ArrayRef<StringRef> Filenames;
public:
- CoverageFilenamesSectionWriter(ArrayRef<StringRef> Filenames)
- : Filenames(Filenames) {}
+ CoverageFilenamesSectionWriter(ArrayRef<StringRef> Filenames);
- /// Write encoded filenames to the given output stream.
- void write(raw_ostream &OS);
+ /// Write encoded filenames to the given output stream. If \p Compress is
+ /// true, attempt to compress the filenames.
+ void write(raw_ostream &OS, bool Compress = true);
};
/// Writer for instrumentation based coverage mapping data.
diff --git a/linux-x64/clang/include/llvm/ProfileData/GCOV.h b/linux-x64/clang/include/llvm/ProfileData/GCOV.h
index 004ff3f..d4f0b91 100644
--- a/linux-x64/clang/include/llvm/ProfileData/GCOV.h
+++ b/linux-x64/clang/include/llvm/ProfileData/GCOV.h
@@ -15,12 +15,14 @@
#define LLVM_PROFILEDATA_GCOV_H
#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/iterator.h"
#include "llvm/ADT/iterator_range.h"
+#include "llvm/Support/DataExtractor.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
@@ -28,6 +30,7 @@
#include <cstddef>
#include <cstdint>
#include <limits>
+#include <map>
#include <memory>
#include <string>
#include <utility>
@@ -36,18 +39,19 @@
class GCOVFunction;
class GCOVBlock;
-class FileInfo;
namespace GCOV {
-enum GCOVVersion { V402, V404, V704 };
+enum GCOVVersion { V304, V407, V408, V800, V900 };
/// A struct for passing gcov options between functions.
struct Options {
- Options(bool A, bool B, bool C, bool F, bool P, bool U, bool L, bool N, bool X)
+ Options(bool A, bool B, bool C, bool F, bool P, bool U, bool I, bool L,
+ bool M, bool N, bool R, bool T, bool X, std::string SourcePrefix)
: AllBlocks(A), BranchInfo(B), BranchCount(C), FuncCoverage(F),
- PreservePaths(P), UncondBranch(U), LongFileNames(L), NoOutput(N),
- HashFilenames(X) {}
+ PreservePaths(P), UncondBranch(U), Intermediate(I), LongFileNames(L),
+ Demangle(M), NoOutput(N), RelativeOnly(R), UseStdout(T),
+ HashFilenames(X), SourcePrefix(std::move(SourcePrefix)) {}
bool AllBlocks;
bool BranchInfo;
@@ -55,9 +59,14 @@
bool FuncCoverage;
bool PreservePaths;
bool UncondBranch;
+ bool Intermediate;
bool LongFileNames;
+ bool Demangle;
bool NoOutput;
+ bool RelativeOnly;
+ bool UseStdout;
bool HashFilenames;
+ std::string SourcePrefix;
};
} // end namespace GCOV
@@ -67,143 +76,86 @@
class GCOVBuffer {
public:
GCOVBuffer(MemoryBuffer *B) : Buffer(B) {}
+ ~GCOVBuffer() { consumeError(cursor.takeError()); }
/// readGCNOFormat - Check GCNO signature is valid at the beginning of buffer.
bool readGCNOFormat() {
- StringRef File = Buffer->getBuffer().slice(0, 4);
- if (File != "oncg") {
- errs() << "Unexpected file type: " << File << ".\n";
+ StringRef buf = Buffer->getBuffer();
+ StringRef magic = buf.substr(0, 4);
+ if (magic == "gcno") {
+ de = DataExtractor(buf.substr(4), false, 0);
+ } else if (magic == "oncg") {
+ de = DataExtractor(buf.substr(4), true, 0);
+ } else {
+ errs() << "unexpected magic: " << magic << "\n";
return false;
}
- Cursor = 4;
return true;
}
/// readGCDAFormat - Check GCDA signature is valid at the beginning of buffer.
bool readGCDAFormat() {
- StringRef File = Buffer->getBuffer().slice(0, 4);
- if (File != "adcg") {
- errs() << "Unexpected file type: " << File << ".\n";
+ StringRef buf = Buffer->getBuffer();
+ StringRef magic = buf.substr(0, 4);
+ if (magic == "gcda") {
+ de = DataExtractor(buf.substr(4), false, 0);
+ } else if (magic == "adcg") {
+ de = DataExtractor(buf.substr(4), true, 0);
+ } else {
return false;
}
- Cursor = 4;
return true;
}
/// readGCOVVersion - Read GCOV version.
bool readGCOVVersion(GCOV::GCOVVersion &Version) {
- StringRef VersionStr = Buffer->getBuffer().slice(Cursor, Cursor + 4);
- if (VersionStr == "*204") {
- Cursor += 4;
- Version = GCOV::V402;
+ std::string str(de.getBytes(cursor, 4));
+ if (str.size() != 4)
+ return false;
+ if (de.isLittleEndian())
+ std::reverse(str.begin(), str.end());
+ int ver = str[0] >= 'A'
+ ? (str[0] - 'A') * 100 + (str[1] - '0') * 10 + str[2] - '0'
+ : (str[0] - '0') * 10 + str[2] - '0';
+ if (ver >= 90) {
+ // PR gcov-profile/84846, r269678
+ Version = GCOV::V900;
+ return true;
+ } else if (ver >= 80) {
+ // PR gcov-profile/48463
+ Version = GCOV::V800;
+ return true;
+ } else if (ver >= 48) {
+ // r189778: the exit block moved from the last to the second.
+ Version = GCOV::V408;
+ return true;
+ } else if (ver >= 47) {
+ // r173147: split checksum into cfg checksum and line checksum.
+ Version = GCOV::V407;
+ return true;
+ } else if (ver >= 34) {
+ Version = GCOV::V304;
return true;
}
- if (VersionStr == "*404") {
- Cursor += 4;
- Version = GCOV::V404;
- return true;
- }
- if (VersionStr == "*704") {
- Cursor += 4;
- Version = GCOV::V704;
- return true;
- }
- errs() << "Unexpected version: " << VersionStr << ".\n";
+ errs() << "unexpected version: " << str << "\n";
return false;
}
- /// readFunctionTag - If cursor points to a function tag then increment the
- /// cursor and return true otherwise return false.
- bool readFunctionTag() {
- StringRef Tag = Buffer->getBuffer().slice(Cursor, Cursor + 4);
- if (Tag.empty() || Tag[0] != '\0' || Tag[1] != '\0' || Tag[2] != '\0' ||
- Tag[3] != '\1') {
- return false;
- }
- Cursor += 4;
- return true;
- }
-
- /// readBlockTag - If cursor points to a block tag then increment the
- /// cursor and return true otherwise return false.
- bool readBlockTag() {
- StringRef Tag = Buffer->getBuffer().slice(Cursor, Cursor + 4);
- if (Tag.empty() || Tag[0] != '\0' || Tag[1] != '\0' || Tag[2] != '\x41' ||
- Tag[3] != '\x01') {
- return false;
- }
- Cursor += 4;
- return true;
- }
-
- /// readEdgeTag - If cursor points to an edge tag then increment the
- /// cursor and return true otherwise return false.
- bool readEdgeTag() {
- StringRef Tag = Buffer->getBuffer().slice(Cursor, Cursor + 4);
- if (Tag.empty() || Tag[0] != '\0' || Tag[1] != '\0' || Tag[2] != '\x43' ||
- Tag[3] != '\x01') {
- return false;
- }
- Cursor += 4;
- return true;
- }
-
- /// readLineTag - If cursor points to a line tag then increment the
- /// cursor and return true otherwise return false.
- bool readLineTag() {
- StringRef Tag = Buffer->getBuffer().slice(Cursor, Cursor + 4);
- if (Tag.empty() || Tag[0] != '\0' || Tag[1] != '\0' || Tag[2] != '\x45' ||
- Tag[3] != '\x01') {
- return false;
- }
- Cursor += 4;
- return true;
- }
-
- /// readArcTag - If cursor points to an gcda arc tag then increment the
- /// cursor and return true otherwise return false.
- bool readArcTag() {
- StringRef Tag = Buffer->getBuffer().slice(Cursor, Cursor + 4);
- if (Tag.empty() || Tag[0] != '\0' || Tag[1] != '\0' || Tag[2] != '\xa1' ||
- Tag[3] != '\1') {
- return false;
- }
- Cursor += 4;
- return true;
- }
-
- /// readObjectTag - If cursor points to an object summary tag then increment
- /// the cursor and return true otherwise return false.
- bool readObjectTag() {
- StringRef Tag = Buffer->getBuffer().slice(Cursor, Cursor + 4);
- if (Tag.empty() || Tag[0] != '\0' || Tag[1] != '\0' || Tag[2] != '\0' ||
- Tag[3] != '\xa1') {
- return false;
- }
- Cursor += 4;
- return true;
- }
-
- /// readProgramTag - If cursor points to a program summary tag then increment
- /// the cursor and return true otherwise return false.
- bool readProgramTag() {
- StringRef Tag = Buffer->getBuffer().slice(Cursor, Cursor + 4);
- if (Tag.empty() || Tag[0] != '\0' || Tag[1] != '\0' || Tag[2] != '\0' ||
- Tag[3] != '\xa3') {
- return false;
- }
- Cursor += 4;
- return true;
+ uint32_t getWord() { return de.getU32(cursor); }
+ StringRef getString() {
+ uint32_t len;
+ if (!readInt(len) || len == 0)
+ return {};
+ return de.getBytes(cursor, len * 4).split('\0').first;
}
bool readInt(uint32_t &Val) {
- if (Buffer->getBuffer().size() < Cursor + 4) {
- errs() << "Unexpected end of memory buffer: " << Cursor + 4 << ".\n";
+ if (cursor.tell() + 4 > de.size()) {
+ Val = 0;
+ errs() << "unexpected end of memory buffer: " << cursor.tell() << "\n";
return false;
}
- StringRef Str = Buffer->getBuffer().slice(Cursor, Cursor + 4);
- Cursor += 4;
- Val = *(const uint32_t *)(Str.data());
+ Val = de.getU32(cursor);
return true;
}
@@ -216,28 +168,18 @@
}
bool readString(StringRef &Str) {
- uint32_t Len = 0;
- // Keep reading until we find a non-zero length. This emulates gcov's
- // behaviour, which appears to do the same.
- while (Len == 0)
- if (!readInt(Len))
- return false;
- Len *= 4;
- if (Buffer->getBuffer().size() < Cursor + Len) {
- errs() << "Unexpected end of memory buffer: " << Cursor + Len << ".\n";
+ uint32_t len;
+ if (!readInt(len) || len == 0)
return false;
- }
- Str = Buffer->getBuffer().slice(Cursor, Cursor + Len).split('\0').first;
- Cursor += Len;
- return true;
+ Str = de.getBytes(cursor, len * 4).split('\0').first;
+ return bool(cursor);
}
- uint64_t getCursor() const { return Cursor; }
- void advanceCursor(uint32_t n) { Cursor += n * 4; }
+ DataExtractor de{ArrayRef<uint8_t>{}, false, 0};
+ DataExtractor::Cursor cursor{0};
private:
MemoryBuffer *Buffer;
- uint64_t Cursor = 0;
};
/// GCOVFile - Collects coverage information for one pair of coverage file
@@ -248,28 +190,39 @@
bool readGCNO(GCOVBuffer &Buffer);
bool readGCDA(GCOVBuffer &Buffer);
- uint32_t getChecksum() const { return Checksum; }
+ GCOV::GCOVVersion getVersion() const { return Version; }
void print(raw_ostream &OS) const;
void dump() const;
- void collectLineCounts(FileInfo &FI);
-private:
+ std::vector<std::string> filenames;
+ StringMap<unsigned> filenameToIdx;
+
+public:
bool GCNOInitialized = false;
GCOV::GCOVVersion Version;
uint32_t Checksum = 0;
- SmallVector<std::unique_ptr<GCOVFunction>, 16> Functions;
+ StringRef cwd;
+ SmallVector<std::unique_ptr<GCOVFunction>, 16> functions;
+ std::map<uint32_t, GCOVFunction *> IdentToFunction;
uint32_t RunCount = 0;
uint32_t ProgramCount = 0;
+
+ using iterator = pointee_iterator<
+ SmallVectorImpl<std::unique_ptr<GCOVFunction>>::const_iterator>;
+ iterator begin() const { return iterator(functions.begin()); }
+ iterator end() const { return iterator(functions.end()); }
};
-/// GCOVEdge - Collects edge information.
-struct GCOVEdge {
- GCOVEdge(GCOVBlock &S, GCOVBlock &D) : Src(S), Dst(D) {}
+struct GCOVArc {
+ GCOVArc(GCOVBlock &src, GCOVBlock &dst, uint32_t flags)
+ : src(src), dst(dst), flags(flags) {}
+ bool onTree() const;
- GCOVBlock &Src;
- GCOVBlock &Dst;
- uint64_t Count = 0;
- uint64_t CyclesCount = 0;
+ GCOVBlock &src;
+ GCOVBlock &dst;
+ uint32_t flags;
+ uint64_t count = 0;
+ uint64_t cycleCount = 0;
};
/// GCOVFunction - Collects function information.
@@ -278,195 +231,86 @@
using BlockIterator = pointee_iterator<
SmallVectorImpl<std::unique_ptr<GCOVBlock>>::const_iterator>;
- GCOVFunction(GCOVFile &P) : Parent(P) {}
+ GCOVFunction(GCOVFile &file) : file(file) {}
- bool readGCNO(GCOVBuffer &Buffer, GCOV::GCOVVersion Version);
- bool readGCDA(GCOVBuffer &Buffer, GCOV::GCOVVersion Version);
- StringRef getName() const { return Name; }
- StringRef getFilename() const { return Filename; }
- size_t getNumBlocks() const { return Blocks.size(); }
+ StringRef getName(bool demangle) const;
+ StringRef getFilename() const;
uint64_t getEntryCount() const;
- uint64_t getExitCount() const;
+ GCOVBlock &getExitBlock() const;
- BlockIterator block_begin() const { return Blocks.begin(); }
- BlockIterator block_end() const { return Blocks.end(); }
- iterator_range<BlockIterator> blocks() const {
- return make_range(block_begin(), block_end());
+ iterator_range<BlockIterator> blocksRange() const {
+ return make_range(blocks.begin(), blocks.end());
}
+ uint64_t propagateCounts(const GCOVBlock &v, GCOVArc *pred);
void print(raw_ostream &OS) const;
void dump() const;
- void collectLineCounts(FileInfo &FI);
-private:
- GCOVFile &Parent;
- uint32_t Ident = 0;
- uint32_t Checksum;
- uint32_t LineNumber = 0;
+ GCOVFile &file;
+ uint32_t ident = 0;
+ uint32_t linenoChecksum;
+ uint32_t cfgChecksum = 0;
+ uint32_t startLine = 0;
+ uint32_t startColumn = 0;
+ uint32_t endLine = 0;
+ uint32_t endColumn = 0;
+ uint8_t artificial = 0;
StringRef Name;
- StringRef Filename;
- SmallVector<std::unique_ptr<GCOVBlock>, 16> Blocks;
- SmallVector<std::unique_ptr<GCOVEdge>, 16> Edges;
+ mutable SmallString<0> demangled;
+ unsigned srcIdx;
+ SmallVector<std::unique_ptr<GCOVBlock>, 0> blocks;
+ SmallVector<std::unique_ptr<GCOVArc>, 0> arcs, treeArcs;
+ DenseSet<const GCOVBlock *> visited;
};
/// GCOVBlock - Collects block information.
class GCOVBlock {
- struct EdgeWeight {
- EdgeWeight(GCOVBlock *D) : Dst(D) {}
-
- GCOVBlock *Dst;
- uint64_t Count = 0;
- };
-
public:
- using EdgeIterator = SmallVectorImpl<GCOVEdge *>::const_iterator;
- using BlockVector = SmallVector<const GCOVBlock *, 4>;
+ using EdgeIterator = SmallVectorImpl<GCOVArc *>::const_iterator;
+ using BlockVector = SmallVector<const GCOVBlock *, 1>;
using BlockVectorLists = SmallVector<BlockVector, 4>;
- using Edges = SmallVector<GCOVEdge *, 4>;
+ using Edges = SmallVector<GCOVArc *, 4>;
- GCOVBlock(GCOVFunction &P, uint32_t N) : Parent(P), Number(N) {}
- ~GCOVBlock();
+ GCOVBlock(uint32_t N) : number(N) {}
- const GCOVFunction &getParent() const { return Parent; }
- void addLine(uint32_t N) { Lines.push_back(N); }
- uint32_t getLastLine() const { return Lines.back(); }
- void addCount(size_t DstEdgeNo, uint64_t N);
- uint64_t getCount() const { return Counter; }
+ void addLine(uint32_t N) { lines.push_back(N); }
+ uint32_t getLastLine() const { return lines.back(); }
+ uint64_t getCount() const { return count; }
- void addSrcEdge(GCOVEdge *Edge) {
- assert(&Edge->Dst == this); // up to caller to ensure edge is valid
- SrcEdges.push_back(Edge);
- }
+ void addSrcEdge(GCOVArc *Edge) { pred.push_back(Edge); }
- void addDstEdge(GCOVEdge *Edge) {
- assert(&Edge->Src == this); // up to caller to ensure edge is valid
- // Check if adding this edge causes list to become unsorted.
- if (DstEdges.size() && DstEdges.back()->Dst.Number > Edge->Dst.Number)
- DstEdgesAreSorted = false;
- DstEdges.push_back(Edge);
- }
+ void addDstEdge(GCOVArc *Edge) { succ.push_back(Edge); }
- size_t getNumSrcEdges() const { return SrcEdges.size(); }
- size_t getNumDstEdges() const { return DstEdges.size(); }
- void sortDstEdges();
-
- EdgeIterator src_begin() const { return SrcEdges.begin(); }
- EdgeIterator src_end() const { return SrcEdges.end(); }
iterator_range<EdgeIterator> srcs() const {
- return make_range(src_begin(), src_end());
+ return make_range(pred.begin(), pred.end());
}
- EdgeIterator dst_begin() const { return DstEdges.begin(); }
- EdgeIterator dst_end() const { return DstEdges.end(); }
iterator_range<EdgeIterator> dsts() const {
- return make_range(dst_begin(), dst_end());
+ return make_range(succ.begin(), succ.end());
}
void print(raw_ostream &OS) const;
void dump() const;
- void collectLineCounts(FileInfo &FI);
- static uint64_t getCycleCount(const Edges &Path);
- static void unblock(const GCOVBlock *U, BlockVector &Blocked,
- BlockVectorLists &BlockLists);
- static bool lookForCircuit(const GCOVBlock *V, const GCOVBlock *Start,
- Edges &Path, BlockVector &Blocked,
- BlockVectorLists &BlockLists,
- const BlockVector &Blocks, uint64_t &Count);
- static void getCyclesCount(const BlockVector &Blocks, uint64_t &Count);
+ static uint64_t
+ augmentOneCycle(GCOVBlock *src,
+ std::vector<std::pair<GCOVBlock *, size_t>> &stack);
+ static uint64_t getCyclesCount(const BlockVector &blocks);
static uint64_t getLineCount(const BlockVector &Blocks);
-private:
- GCOVFunction &Parent;
- uint32_t Number;
- uint64_t Counter = 0;
- bool DstEdgesAreSorted = true;
- SmallVector<GCOVEdge *, 16> SrcEdges;
- SmallVector<GCOVEdge *, 16> DstEdges;
- SmallVector<uint32_t, 16> Lines;
-};
-
-class FileInfo {
-protected:
- // It is unlikely--but possible--for multiple functions to be on the same
- // line.
- // Therefore this typedef allows LineData.Functions to store multiple
- // functions
- // per instance. This is rare, however, so optimize for the common case.
- using FunctionVector = SmallVector<const GCOVFunction *, 1>;
- using FunctionLines = DenseMap<uint32_t, FunctionVector>;
- using BlockVector = SmallVector<const GCOVBlock *, 4>;
- using BlockLines = DenseMap<uint32_t, BlockVector>;
-
- struct LineData {
- LineData() = default;
-
- BlockLines Blocks;
- FunctionLines Functions;
- uint32_t LastLine = 0;
- };
-
- struct GCOVCoverage {
- GCOVCoverage(StringRef Name) : Name(Name) {}
-
- StringRef Name;
-
- uint32_t LogicalLines = 0;
- uint32_t LinesExec = 0;
-
- uint32_t Branches = 0;
- uint32_t BranchesExec = 0;
- uint32_t BranchesTaken = 0;
- };
-
public:
- FileInfo(const GCOV::Options &Options) : Options(Options) {}
-
- void addBlockLine(StringRef Filename, uint32_t Line, const GCOVBlock *Block) {
- if (Line > LineInfo[Filename].LastLine)
- LineInfo[Filename].LastLine = Line;
- LineInfo[Filename].Blocks[Line - 1].push_back(Block);
- }
-
- void addFunctionLine(StringRef Filename, uint32_t Line,
- const GCOVFunction *Function) {
- if (Line > LineInfo[Filename].LastLine)
- LineInfo[Filename].LastLine = Line;
- LineInfo[Filename].Functions[Line - 1].push_back(Function);
- }
-
- void setRunCount(uint32_t Runs) { RunCount = Runs; }
- void setProgramCount(uint32_t Programs) { ProgramCount = Programs; }
- void print(raw_ostream &OS, StringRef MainFilename, StringRef GCNOFile,
- StringRef GCDAFile);
-
-protected:
- std::string getCoveragePath(StringRef Filename, StringRef MainFilename);
- std::unique_ptr<raw_ostream> openCoveragePath(StringRef CoveragePath);
- void printFunctionSummary(raw_ostream &OS, const FunctionVector &Funcs) const;
- void printBlockInfo(raw_ostream &OS, const GCOVBlock &Block,
- uint32_t LineIndex, uint32_t &BlockNo) const;
- void printBranchInfo(raw_ostream &OS, const GCOVBlock &Block,
- GCOVCoverage &Coverage, uint32_t &EdgeNo);
- void printUncondBranchInfo(raw_ostream &OS, uint32_t &EdgeNo,
- uint64_t Count) const;
-
- void printCoverage(raw_ostream &OS, const GCOVCoverage &Coverage) const;
- void printFuncCoverage(raw_ostream &OS) const;
- void printFileCoverage(raw_ostream &OS) const;
-
- const GCOV::Options &Options;
- StringMap<LineData> LineInfo;
- uint32_t RunCount = 0;
- uint32_t ProgramCount = 0;
-
- using FileCoverageList = SmallVector<std::pair<std::string, GCOVCoverage>, 4>;
- using FuncCoverageMap = MapVector<const GCOVFunction *, GCOVCoverage>;
-
- FileCoverageList FileCoverages;
- FuncCoverageMap FuncCoverages;
+ uint32_t number;
+ uint64_t count = 0;
+ SmallVector<GCOVArc *, 2> pred;
+ SmallVector<GCOVArc *, 2> succ;
+ SmallVector<uint32_t, 4> lines;
+ bool traversable = false;
+ GCOVArc *incoming = nullptr;
};
+void gcovOneInput(const GCOV::Options &options, StringRef filename,
+ StringRef gcno, StringRef gcda, GCOVFile &file);
+
} // end namespace llvm
#endif // LLVM_SUPPORT_GCOV_H
diff --git a/linux-x64/clang/include/llvm/ProfileData/InstrProf.h b/linux-x64/clang/include/llvm/ProfileData/InstrProf.h
index c7d764a..9c16c35 100644
--- a/linux-x64/clang/include/llvm/ProfileData/InstrProf.h
+++ b/linux-x64/clang/include/llvm/ProfileData/InstrProf.h
@@ -23,6 +23,7 @@
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/ProfileSummary.h"
#include "llvm/ProfileData/InstrProfData.inc"
+#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Endian.h"
#include "llvm/Support/Error.h"
@@ -73,9 +74,10 @@
return INSTR_PROF_VALUE_PROF_FUNC_STR;
}
-/// Return the name profile runtime entry point to do value range profiling.
-inline StringRef getInstrProfValueRangeProfFuncName() {
- return INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR;
+/// Return the name profile runtime entry point to do memop size value
+/// profiling.
+inline StringRef getInstrProfValueProfMemOpFuncName() {
+ return INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR;
}
/// Return the name prefix of variables containing instrumented function names.
@@ -93,10 +95,6 @@
/// Return the name of value profile node array variables:
inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes"; }
-/// Return the name prefix of the COMDAT group for instrumentation variables
-/// associated with a COMDAT function.
-inline StringRef getInstrProfComdatPrefix() { return "__profv_"; }
-
/// Return the name of the variable holding the strings (possibly compressed)
/// of all function's PGO names.
inline StringRef getInstrProfNamesVarName() {
@@ -157,6 +155,10 @@
return "__llvm_profile_runtime_user";
}
+inline StringRef getInstrProfCounterBiasVarName() {
+ return "__llvm_profile_counter_bias";
+}
+
/// Return the marker used to separate PGO names during serialization.
inline StringRef getInstrProfNameSeparator() { return "\01"; }
@@ -560,10 +562,9 @@
StringRef InstrProfSymtab::getFuncName(uint64_t FuncMD5Hash) {
finalizeSymtab();
- auto Result =
- std::lower_bound(MD5NameMap.begin(), MD5NameMap.end(), FuncMD5Hash,
- [](const std::pair<uint64_t, std::string> &LHS,
- uint64_t RHS) { return LHS.first < RHS; });
+ auto Result = llvm::lower_bound(MD5NameMap, FuncMD5Hash,
+ [](const std::pair<uint64_t, StringRef> &LHS,
+ uint64_t RHS) { return LHS.first < RHS; });
if (Result != MD5NameMap.end() && Result->first == FuncMD5Hash)
return Result->second;
return StringRef();
@@ -571,10 +572,9 @@
Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) {
finalizeSymtab();
- auto Result =
- std::lower_bound(MD5FuncMap.begin(), MD5FuncMap.end(), FuncMD5Hash,
- [](const std::pair<uint64_t, Function*> &LHS,
- uint64_t RHS) { return LHS.first < RHS; });
+ auto Result = llvm::lower_bound(MD5FuncMap, FuncMD5Hash,
+ [](const std::pair<uint64_t, Function *> &LHS,
+ uint64_t RHS) { return LHS.first < RHS; });
if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash)
return Result->second;
return nullptr;
@@ -634,8 +634,8 @@
FuncHash = Hash;
}
- Error accumuateCounts(const std::string &BaseFilename,
- const std::string &TestFilename, bool IsCS);
+ Error accumulateCounts(const std::string &BaseFilename,
+ const std::string &TestFilename, bool IsCS);
void addOneMismatch(const CountSumOrPercent &MismatchFunc);
void addOneUnique(const CountSumOrPercent &UniqueFunc);
@@ -677,8 +677,8 @@
/// Optionally scale merged counts by \p Weight.
void merge(InstrProfValueSiteRecord &Input, uint64_t Weight,
function_ref<void(instrprof_error)> Warn);
- /// Scale up value profile data counts.
- void scale(uint64_t Weight, function_ref<void(instrprof_error)> Warn);
+ /// Scale up value profile data counts by N (Numerator) / D (Denominator).
+ void scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn);
/// Compute the overlap b/w this record and Input record.
void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind,
@@ -695,7 +695,7 @@
InstrProfRecord(const InstrProfRecord &RHS)
: Counts(RHS.Counts),
ValueData(RHS.ValueData
- ? llvm::make_unique<ValueProfData>(*RHS.ValueData)
+ ? std::make_unique<ValueProfData>(*RHS.ValueData)
: nullptr) {}
InstrProfRecord &operator=(InstrProfRecord &&) = default;
InstrProfRecord &operator=(const InstrProfRecord &RHS) {
@@ -705,7 +705,7 @@
return *this;
}
if (!ValueData)
- ValueData = llvm::make_unique<ValueProfData>(*RHS.ValueData);
+ ValueData = std::make_unique<ValueProfData>(*RHS.ValueData);
else
*ValueData = *RHS.ValueData;
return *this;
@@ -752,8 +752,8 @@
function_ref<void(instrprof_error)> Warn);
/// Scale up profile counts (including value profile data) by
- /// \p Weight.
- void scale(uint64_t Weight, function_ref<void(instrprof_error)> Warn);
+ /// a factor of (N / D).
+ void scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn);
/// Sort value profile data (per site) by count.
void sortValueData() {
@@ -772,7 +772,7 @@
void clearValueData() { ValueData = nullptr; }
/// Compute the sums of all counts and store in Sum.
- void accumuateCounts(CountSumOrPercent &Sum) const;
+ void accumulateCounts(CountSumOrPercent &Sum) const;
/// Compute the overlap b/w this IntrprofRecord and Other.
void overlap(InstrProfRecord &Other, OverlapStats &Overlap,
@@ -817,7 +817,7 @@
std::vector<InstrProfValueSiteRecord> &
getOrCreateValueSitesForKind(uint32_t ValueKind) {
if (!ValueData)
- ValueData = llvm::make_unique<ValueProfData>();
+ ValueData = std::make_unique<ValueProfData>();
switch (ValueKind) {
case IPVK_IndirectCallTarget:
return ValueData->IndirectCallSites;
@@ -838,8 +838,8 @@
uint64_t Weight,
function_ref<void(instrprof_error)> Warn);
- // Scale up value profile data count.
- void scaleValueProfData(uint32_t ValueKind, uint64_t Weight,
+ // Scale up value profile data count by N (Numerator) / D (Denominator).
+ void scaleValueProfData(uint32_t ValueKind, uint64_t N, uint64_t D,
function_ref<void(instrprof_error)> Warn);
};
@@ -889,7 +889,7 @@
std::unique_ptr<InstrProfValueData[]>
InstrProfRecord::getValueForSite(uint32_t ValueKind, uint32_t Site,
uint64_t *TotalC) const {
- uint64_t Dummy;
+ uint64_t Dummy = 0;
uint64_t &TotalCount = (TotalC == nullptr ? Dummy : *TotalC);
uint32_t N = getNumValueDataForSite(ValueKind, Site);
if (N == 0) {
@@ -897,7 +897,7 @@
return std::unique_ptr<InstrProfValueData[]>(nullptr);
}
- auto VD = llvm::make_unique<InstrProfValueData[]>(N);
+ auto VD = std::make_unique<InstrProfValueData[]>(N);
TotalCount = getValueForSite(VD.get(), ValueKind, Site);
return VD;
@@ -978,7 +978,12 @@
Version4 = 4,
// In this version, the frontend PGO stable hash algorithm defaults to V2.
Version5 = 5,
- // The current version is 5.
+ // In this version, the frontend PGO stable hash algorithm got fixed and
+ // may produce hashes different from Version5.
+ Version6 = 6,
+ // An additional counter is added around logical operators.
+ Version7 = 7,
+ // The current version is 7.
CurrentVersion = INSTR_PROF_INDEX_VERSION
};
const uint64_t Version = ProfVersion::CurrentVersion;
@@ -1134,10 +1139,15 @@
// Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime
// aware this is an ir_level profile so it can set the version flag.
-void createIRLevelProfileFlagVar(Module &M, bool IsCS);
+void createIRLevelProfileFlagVar(Module &M, bool IsCS,
+ bool InstrEntryBBEnabled);
// Create the variable for the profile file name.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput);
+// Whether to compress function names in profile records, and filenames in
+// code coverage mappings. Used by the Instrumentation library and unit tests.
+extern cl::opt<bool> DoInstrProfNameCompression;
+
} // end namespace llvm
#endif // LLVM_PROFILEDATA_INSTRPROF_H
diff --git a/linux-x64/clang/include/llvm/ProfileData/InstrProfData.inc b/linux-x64/clang/include/llvm/ProfileData/InstrProfData.inc
index 749781b..f715505 100644
--- a/linux-x64/clang/include/llvm/ProfileData/InstrProfData.inc
+++ b/linux-x64/clang/include/llvm/ProfileData/InstrProfData.inc
@@ -130,7 +130,9 @@
INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize)
+INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters)
INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize)
+INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters)
INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
@@ -152,17 +154,7 @@
VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \
INSTR_PROF_COMMA
VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA
-#ifndef VALUE_RANGE_PROF
VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
-#else /* VALUE_RANGE_PROF */
-VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) \
- INSTR_PROF_COMMA
-VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeStart, Type::getInt64Ty(Ctx)) \
- INSTR_PROF_COMMA
-VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeLast, Type::getInt64Ty(Ctx)) \
- INSTR_PROF_COMMA
-VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx))
-#endif /*VALUE_RANGE_PROF */
#undef VALUE_PROF_FUNC_PARAM
#undef INSTR_PROF_COMMA
/* VALUE_PROF_FUNC_PARAM end */
@@ -196,6 +188,14 @@
#undef VALUE_PROF_KIND
/* VALUE_PROF_KIND end */
+#undef COVMAP_V2_OR_V3
+#ifdef COVMAP_V2
+#define COVMAP_V2_OR_V3
+#endif
+#ifdef COVMAP_V3
+#define COVMAP_V2_OR_V3
+#endif
+
/* COVMAP_FUNC_RECORD start */
/* Definition of member fields of the function record structure in coverage
* map.
@@ -212,16 +212,30 @@
COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \
llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \
NameValue.size()))
-#else
+#endif
+#ifdef COVMAP_V2_OR_V3
COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
- llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
- llvm::IndexedInstrProf::ComputeHash(NameValue)))
+ llvm::ConstantInt::get( \
+ llvm::Type::getInt64Ty(Ctx), NameHash))
#endif
COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \
- llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\
- CoverageMapping.size()))
+ llvm::ConstantInt::get( \
+ llvm::Type::getInt32Ty(Ctx), CoverageMapping.size()))
COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
- llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), FuncHash))
+ llvm::ConstantInt::get( \
+ llvm::Type::getInt64Ty(Ctx), FuncHash))
+#ifdef COVMAP_V3
+COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \
+ llvm::ConstantInt::get( \
+ llvm::Type::getInt64Ty(Ctx), FilenamesRef))
+COVMAP_FUNC_RECORD(const char, \
+ llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \
+ CoverageMapping.size()), \
+ CoverageMapping,
+ llvm::ConstantDataArray::getRaw( \
+ CoverageMapping, CoverageMapping.size(), \
+ llvm::Type::getInt8Ty(Ctx)))
+#endif
#undef COVMAP_FUNC_RECORD
/* COVMAP_FUNC_RECORD end. */
@@ -234,7 +248,7 @@
#define INSTR_PROF_DATA_DEFINED
#endif
COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \
- llvm::ConstantInt::get(Int32Ty, FunctionRecords.size()))
+ llvm::ConstantInt::get(Int32Ty, NRecords))
COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \
llvm::ConstantInt::get(Int32Ty, FilenamesSize))
COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \
@@ -265,6 +279,9 @@
INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
+INSTR_PROF_SECT_ENTRY(IPSK_covfun, \
+ INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \
+ INSTR_PROF_COVFUN_COFF, "__LLVM_COV,")
INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \
INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \
INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,")
@@ -628,11 +645,11 @@
(uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
/* Raw profile format version (start from 1). */
-#define INSTR_PROF_RAW_VERSION 4
+#define INSTR_PROF_RAW_VERSION 5
/* Indexed profile format version (start from 1). */
-#define INSTR_PROF_INDEX_VERSION 5
-/* Coverage mapping format vresion (start from 0). */
-#define INSTR_PROF_COVMAP_VERSION 2
+#define INSTR_PROF_INDEX_VERSION 7
+/* Coverage mapping format version (start from 0). */
+#define INSTR_PROF_COVMAP_VERSION 4
/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the
* version for other variants of profile. We set the lowest bit of the upper 8
@@ -644,6 +661,7 @@
#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
#define VARIANT_MASK_IR_PROF (0x1ULL << 56)
#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
+#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
@@ -659,6 +677,7 @@
#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
#define INSTR_PROF_COVMAP_COMMON __llvm_covmap
+#define INSTR_PROF_COVFUN_COMMON __llvm_covfun
#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
/* Windows section names. Because these section names contain dollar characters,
* they must be quoted.
@@ -669,6 +688,7 @@
#define INSTR_PROF_VALS_COFF ".lprfv$M"
#define INSTR_PROF_VNODES_COFF ".lprfnd$M"
#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
+#define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
#ifdef _WIN32
@@ -683,6 +703,7 @@
/* Value profile nodes section. */
#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
+#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF
#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF
#else
/* Runtime section names and name strings. */
@@ -696,6 +717,7 @@
/* Value profile nodes section. */
#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
+#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON)
/* Order file instrumentation. */
#define INSTR_PROF_ORDERFILE_SECT_NAME \
INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON)
@@ -722,9 +744,9 @@
#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
#define INSTR_PROF_VALUE_PROF_FUNC_STR \
INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
-#define INSTR_PROF_VALUE_RANGE_PROF_FUNC __llvm_profile_instrument_range
-#define INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR \
- INSTR_PROF_QUOTE(INSTR_PROF_VALUE_RANGE_PROF_FUNC)
+#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
+#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
+ INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
/* InstrProfile per-function control data alignment. */
#define INSTR_PROF_DATA_ALIGNMENT 8
@@ -742,7 +764,7 @@
#endif /* INSTR_PROF_DATA_INC */
#ifndef INSTR_ORDER_FILE_INC
-// The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB).
+/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */
#define INSTR_ORDER_FILE_BUFFER_SIZE 131072
#define INSTR_ORDER_FILE_BUFFER_BITS 17
#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff
@@ -750,3 +772,123 @@
#else
#undef INSTR_PROF_DATA_DEFINED
#endif
+
+#undef COVMAP_V2_OR_V3
+
+#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
+
+#ifdef __cplusplus
+#define INSTR_PROF_INLINE inline
+#else
+#define INSTR_PROF_INLINE
+#endif
+
+/* The value range buckets (22 buckets) for the memop size value profiling looks
+ * like:
+ *
+ * [0, 0]
+ * [1, 1]
+ * [2, 2]
+ * [3, 3]
+ * [4, 4]
+ * [5, 5]
+ * [6, 6]
+ * [7, 7]
+ * [8, 8]
+ * [9, 15]
+ * [16, 16]
+ * [17, 31]
+ * [32, 32]
+ * [33, 63]
+ * [64, 64]
+ * [65, 127]
+ * [128, 128]
+ * [129, 255]
+ * [256, 256]
+ * [257, 511]
+ * [512, 512]
+ * [513, UINT64_MAX]
+ *
+ * Each range has a 'representative value' which is the lower end value of the
+ * range and used to store in the runtime profile data records and the VP
+ * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
+ */
+
+/*
+ * Clz and Popcount. This code was copied from
+ * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and
+ * llvm/include/llvm/Support/MathExtras.h.
+ */
+#if defined(_MSC_VER) && !defined(__clang__)
+
+#include <intrin.h>
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
+int InstProfClzll(unsigned long long X) {
+ unsigned long LeadZeroIdx = 0;
+#if !defined(_M_ARM64) && !defined(_M_X64)
+ // Scan the high 32 bits.
+ if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))
+ return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset
+ // from the MSB.
+ // Scan the low 32 bits.
+ if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))
+ return (int)(63 - LeadZeroIdx);
+#else
+ if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
+#endif
+ return 64;
+}
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
+int InstProfPopcountll(unsigned long long X) {
+ // This code originates from https://reviews.llvm.org/rG30626254510f.
+ unsigned long long v = X;
+ v = v - ((v >> 1) & 0x5555555555555555ULL);
+ v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
+ v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
+ return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);
+}
+
+#else
+
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
+int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
+int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }
+
+#endif /* defined(_MSC_VER) && !defined(__clang__) */
+
+/* Map an (observed) memop size value to the representative value of its range.
+ * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
+InstrProfGetRangeRepValue(uint64_t Value) {
+ if (Value <= 8)
+ // The first ranges are individually tracked. Use the value as is.
+ return Value;
+ else if (Value >= 513)
+ // The last range is mapped to its lowest value.
+ return 513;
+ else if (InstProfPopcountll(Value) == 1)
+ // If it's a power of two, use it as is.
+ return Value;
+ else
+ // Otherwise, take to the previous power of two + 1.
+ return (1 << (64 - InstProfClzll(Value) - 1)) + 1;
+}
+
+/* Return true if the range that an (observed) memop size value belongs to has
+ * only a single value in the range. For example, 0 -> true, 8 -> true, 10 ->
+ * false, 64 -> true, 100 -> false, 513 -> false. */
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
+InstrProfIsSingleValRange(uint64_t Value) {
+ if (Value <= 8)
+ // The first ranges are individually tracked.
+ return 1;
+ else if (InstProfPopcountll(Value) == 1)
+ // If it's a power of two, there's only one value.
+ return 1;
+ else
+ // Otherwise, there's more than one value in the range.
+ return 0;
+}
+
+#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */
diff --git a/linux-x64/clang/include/llvm/ProfileData/InstrProfReader.h b/linux-x64/clang/include/llvm/ProfileData/InstrProfReader.h
index 73751fa..2c2cfb9 100644
--- a/linux-x64/clang/include/llvm/ProfileData/InstrProfReader.h
+++ b/linux-x64/clang/include/llvm/ProfileData/InstrProfReader.h
@@ -50,8 +50,12 @@
InstrProfIterator(InstrProfReader *Reader) : Reader(Reader) { Increment(); }
InstrProfIterator &operator++() { Increment(); return *this; }
- bool operator==(const InstrProfIterator &RHS) { return Reader == RHS.Reader; }
- bool operator!=(const InstrProfIterator &RHS) { return Reader != RHS.Reader; }
+ bool operator==(const InstrProfIterator &RHS) const {
+ return Reader == RHS.Reader;
+ }
+ bool operator!=(const InstrProfIterator &RHS) const {
+ return Reader != RHS.Reader;
+ }
value_type &operator*() { return Record; }
value_type *operator->() { return &Record; }
};
@@ -79,6 +83,8 @@
virtual bool hasCSIRLevelProfile() const = 0;
+ virtual bool instrEntryBBEnabled() const = 0;
+
/// Return the PGO symtab. There are three different readers:
/// Raw, Text, and Indexed profile readers. The first two types
/// of readers are used only by llvm-profdata tool, while the indexed
@@ -92,7 +98,7 @@
virtual InstrProfSymtab &getSymtab() = 0;
/// Compute the sum of counts and return in Sum.
- void accumuateCounts(CountSumOrPercent &Sum, bool IsCS);
+ void accumulateCounts(CountSumOrPercent &Sum, bool IsCS);
protected:
std::unique_ptr<InstrProfSymtab> Symtab;
@@ -148,6 +154,7 @@
line_iterator Line;
bool IsIRLevelProfile = false;
bool HasCSIRLevelProfile = false;
+ bool InstrEntryBBEnabled = false;
Error readValueProfileData(InstrProfRecord &Record);
@@ -164,6 +171,8 @@
bool hasCSIRLevelProfile() const override { return HasCSIRLevelProfile; }
+ bool instrEntryBBEnabled() const override { return InstrEntryBBEnabled; }
+
/// Read the header.
Error readHeader() override;
@@ -224,6 +233,10 @@
return (Version & VARIANT_MASK_CSIR_PROF) != 0;
}
+ bool instrEntryBBEnabled() const override {
+ return (Version & VARIANT_MASK_INSTR_ENTRY) != 0;
+ }
+
InstrProfSymtab &getSymtab() override {
assert(Symtab.get());
return *Symtab.get();
@@ -268,8 +281,14 @@
return (const char *)ValueDataStart;
}
- const uint64_t *getCounter(IntPtrT CounterPtr) const {
- ptrdiff_t Offset = (swap(CounterPtr) - CountersDelta) / sizeof(uint64_t);
+ /// Get the offset of \p CounterPtr from the start of the counters section of
+ /// the profile. The offset has units of "number of counters", i.e. increasing
+ /// the offset by 1 corresponds to an increase in the *byte offset* by 8.
+ ptrdiff_t getCounterOffset(IntPtrT CounterPtr) const {
+ return (swap(CounterPtr) - CountersDelta) / sizeof(uint64_t);
+ }
+
+ const uint64_t *getCounter(ptrdiff_t Offset) const {
return CountersStart + Offset;
}
@@ -354,6 +373,7 @@
virtual uint64_t getVersion() const = 0;
virtual bool isIRLevelProfile() const = 0;
virtual bool hasCSIRLevelProfile() const = 0;
+ virtual bool instrEntryBBEnabled() const = 0;
virtual Error populateSymtab(InstrProfSymtab &) = 0;
};
@@ -402,6 +422,10 @@
return (FormatVersion & VARIANT_MASK_CSIR_PROF) != 0;
}
+ bool instrEntryBBEnabled() const override {
+ return (FormatVersion & VARIANT_MASK_INSTR_ENTRY) != 0;
+ }
+
Error populateSymtab(InstrProfSymtab &Symtab) override {
return Symtab.create(HashTable->keys());
}
@@ -456,6 +480,10 @@
return Index->hasCSIRLevelProfile();
}
+ bool instrEntryBBEnabled() const override {
+ return Index->instrEntryBBEnabled();
+ }
+
/// Return true if the given buffer is in an indexed instrprof format.
static bool hasFormat(const MemoryBuffer &DataBuffer);
diff --git a/linux-x64/clang/include/llvm/ProfileData/InstrProfWriter.h b/linux-x64/clang/include/llvm/ProfileData/InstrProfWriter.h
index 5882fa2..35c2669 100644
--- a/linux-x64/clang/include/llvm/ProfileData/InstrProfWriter.h
+++ b/linux-x64/clang/include/llvm/ProfileData/InstrProfWriter.h
@@ -40,13 +40,16 @@
bool Sparse;
StringMap<ProfilingData> FunctionData;
ProfKind ProfileKind = PF_Unknown;
+ bool InstrEntryBBEnabled;
// Use raw pointer here for the incomplete type object.
InstrProfRecordWriterTrait *InfoObj;
public:
- InstrProfWriter(bool Sparse = false);
+ InstrProfWriter(bool Sparse = false, bool InstrEntryBBEnabled = false);
~InstrProfWriter();
+ StringMap<ProfilingData> &getProfileData() { return FunctionData; }
+
/// Add function counts for the given function. If there are already counts
/// for this function and the hash and number of counts match, each counter is
/// summed. Optionally scale counts by \p Weight.
@@ -97,6 +100,7 @@
return Error::success();
}
+ void setInstrEntryBBEnabled(bool Enabled) { InstrEntryBBEnabled = Enabled; }
// Internal interface for testing purpose only.
void setValueProfDataEndianness(support::endianness Endianness);
void setOutputSparse(bool Sparse);
diff --git a/linux-x64/clang/include/llvm/ProfileData/ProfileCommon.h b/linux-x64/clang/include/llvm/ProfileData/ProfileCommon.h
index f98a343..6bb5825 100644
--- a/linux-x64/clang/include/llvm/ProfileData/ProfileCommon.h
+++ b/linux-x64/clang/include/llvm/ProfileData/ProfileCommon.h
@@ -33,8 +33,8 @@
} // end namespace sampleprof
-inline const char *getHotSectionPrefix() { return ".hot"; }
-inline const char *getUnlikelySectionPrefix() { return ".unlikely"; }
+inline const char *getHotSectionPrefix() { return "hot"; }
+inline const char *getUnlikelySectionPrefix() { return "unlikely"; }
class ProfileSummaryBuilder {
private:
@@ -62,6 +62,10 @@
public:
/// A vector of useful cutoff values for detailed summary.
static const ArrayRef<uint32_t> DefaultCutoffs;
+
+ /// Find the summary entry for a desired percentile of counts.
+ static const ProfileSummaryEntry &
+ getEntryForPercentile(SummaryEntryVector &DS, uint64_t Percentile);
};
class InstrProfSummaryBuilder final : public ProfileSummaryBuilder {
diff --git a/linux-x64/clang/include/llvm/ProfileData/SampleProf.h b/linux-x64/clang/include/llvm/ProfileData/SampleProf.h
index 7fbc857..c423466 100644
--- a/linux-x64/clang/include/llvm/ProfileData/SampleProf.h
+++ b/linux-x64/clang/include/llvm/ProfileData/SampleProf.h
@@ -18,23 +18,25 @@
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSet.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/Module.h"
+#include "llvm/Support/Allocator.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <cstdint>
#include <map>
+#include <set>
#include <string>
#include <system_error>
#include <utility>
namespace llvm {
-class raw_ostream;
-
const std::error_category &sampleprof_category();
enum class sampleprof_error {
@@ -49,7 +51,11 @@
truncated_name_table,
not_implemented,
counter_overflow,
- ostream_seek_unsupported
+ ostream_seek_unsupported,
+ compress_failed,
+ uncompress_failed,
+ zlib_unavailable,
+ hash_mismatch
};
inline std::error_code make_error_code(sampleprof_error E) {
@@ -83,6 +89,7 @@
SPF_Text = 0x1,
SPF_Compact_Binary = 0x2,
SPF_GCC = 0x3,
+ SPF_Ext_Binary = 0x4,
SPF_Binary = 0xff
};
@@ -93,18 +100,146 @@
uint64_t('2') << (64 - 56) | uint64_t(Format);
}
-// Get the proper representation of a string in the input Format.
-static inline StringRef getRepInFormat(StringRef Name,
- SampleProfileFormat Format,
+/// Get the proper representation of a string according to whether the
+/// current Format uses MD5 to represent the string.
+static inline StringRef getRepInFormat(StringRef Name, bool UseMD5,
std::string &GUIDBuf) {
if (Name.empty())
return Name;
GUIDBuf = std::to_string(Function::getGUID(Name));
- return (Format == SPF_Compact_Binary) ? StringRef(GUIDBuf) : Name;
+ return UseMD5 ? StringRef(GUIDBuf) : Name;
}
static inline uint64_t SPVersion() { return 103; }
+// Section Type used by SampleProfileExtBinaryBaseReader and
+// SampleProfileExtBinaryBaseWriter. Never change the existing
+// value of enum. Only append new ones.
+enum SecType {
+ SecInValid = 0,
+ SecProfSummary = 1,
+ SecNameTable = 2,
+ SecProfileSymbolList = 3,
+ SecFuncOffsetTable = 4,
+ SecFuncMetadata = 5,
+ // marker for the first type of profile.
+ SecFuncProfileFirst = 32,
+ SecLBRProfile = SecFuncProfileFirst
+};
+
+static inline std::string getSecName(SecType Type) {
+ switch (Type) {
+ case SecInValid:
+ return "InvalidSection";
+ case SecProfSummary:
+ return "ProfileSummarySection";
+ case SecNameTable:
+ return "NameTableSection";
+ case SecProfileSymbolList:
+ return "ProfileSymbolListSection";
+ case SecFuncOffsetTable:
+ return "FuncOffsetTableSection";
+ case SecFuncMetadata:
+ return "FunctionMetadata";
+ case SecLBRProfile:
+ return "LBRProfileSection";
+ }
+ llvm_unreachable("A SecType has no name for output");
+}
+
+// Entry type of section header table used by SampleProfileExtBinaryBaseReader
+// and SampleProfileExtBinaryBaseWriter.
+struct SecHdrTableEntry {
+ SecType Type;
+ uint64_t Flags;
+ uint64_t Offset;
+ uint64_t Size;
+ // The index indicating the location of the current entry in
+ // SectionHdrLayout table.
+ uint32_t LayoutIndex;
+};
+
+// Flags common for all sections are defined here. In SecHdrTableEntry::Flags,
+// common flags will be saved in the lower 32bits and section specific flags
+// will be saved in the higher 32 bits.
+enum class SecCommonFlags : uint32_t {
+ SecFlagInValid = 0,
+ SecFlagCompress = (1 << 0)
+};
+
+// Section specific flags are defined here.
+// !!!Note: Everytime a new enum class is created here, please add
+// a new check in verifySecFlag.
+enum class SecNameTableFlags : uint32_t {
+ SecFlagInValid = 0,
+ SecFlagMD5Name = (1 << 0),
+ // Store MD5 in fixed length instead of ULEB128 so NameTable can be
+ // accessed like an array.
+ SecFlagFixedLengthMD5 = (1 << 1)
+};
+enum class SecProfSummaryFlags : uint32_t {
+ SecFlagInValid = 0,
+ /// SecFlagPartial means the profile is for common/shared code.
+ /// The common profile is usually merged from profiles collected
+ /// from running other targets.
+ SecFlagPartial = (1 << 0)
+};
+
+enum class SecFuncMetadataFlags : uint32_t {
+ SecFlagInvalid = 0,
+ SecFlagIsProbeBased = (1 << 0),
+};
+
+// Verify section specific flag is used for the correct section.
+template <class SecFlagType>
+static inline void verifySecFlag(SecType Type, SecFlagType Flag) {
+ // No verification is needed for common flags.
+ if (std::is_same<SecCommonFlags, SecFlagType>())
+ return;
+
+ // Verification starts here for section specific flag.
+ bool IsFlagLegal = false;
+ switch (Type) {
+ case SecNameTable:
+ IsFlagLegal = std::is_same<SecNameTableFlags, SecFlagType>();
+ break;
+ case SecProfSummary:
+ IsFlagLegal = std::is_same<SecProfSummaryFlags, SecFlagType>();
+ break;
+ case SecFuncMetadata:
+ IsFlagLegal = std::is_same<SecFuncMetadataFlags, SecFlagType>();
+ break;
+ default:
+ break;
+ }
+ if (!IsFlagLegal)
+ llvm_unreachable("Misuse of a flag in an incompatible section");
+}
+
+template <class SecFlagType>
+static inline void addSecFlag(SecHdrTableEntry &Entry, SecFlagType Flag) {
+ verifySecFlag(Entry.Type, Flag);
+ auto FVal = static_cast<uint64_t>(Flag);
+ bool IsCommon = std::is_same<SecCommonFlags, SecFlagType>();
+ Entry.Flags |= IsCommon ? FVal : (FVal << 32);
+}
+
+template <class SecFlagType>
+static inline void removeSecFlag(SecHdrTableEntry &Entry, SecFlagType Flag) {
+ verifySecFlag(Entry.Type, Flag);
+ auto FVal = static_cast<uint64_t>(Flag);
+ bool IsCommon = std::is_same<SecCommonFlags, SecFlagType>();
+ Entry.Flags &= ~(IsCommon ? FVal : (FVal << 32));
+}
+
+template <class SecFlagType>
+static inline bool hasSecFlag(const SecHdrTableEntry &Entry, SecFlagType Flag) {
+ verifySecFlag(Entry.Type, Flag);
+ auto FVal = static_cast<uint64_t>(Flag);
+ bool IsCommon = std::is_same<SecCommonFlags, SecFlagType>();
+ return Entry.Flags & (IsCommon ? FVal : (FVal << 32));
+}
+
/// Represents the relative location of an instruction.
///
/// Instruction locations are specified by the line offset from the
@@ -125,6 +260,14 @@
(LineOffset == O.LineOffset && Discriminator < O.Discriminator);
}
+ bool operator==(const LineLocation &O) const {
+ return LineOffset == O.LineOffset && Discriminator == O.Discriminator;
+ }
+
+ bool operator!=(const LineLocation &O) const {
+ return LineOffset != O.LineOffset || Discriminator != O.Discriminator;
+ }
+
uint32_t LineOffset;
uint32_t Discriminator;
};
@@ -143,8 +286,18 @@
/// will be a list of one or more functions.
class SampleRecord {
public:
- using CallTargetMap = StringMap<uint64_t>;
+ using CallTarget = std::pair<StringRef, uint64_t>;
+ struct CallTargetComparator {
+ bool operator()(const CallTarget &LHS, const CallTarget &RHS) const {
+ if (LHS.second != RHS.second)
+ return LHS.second > RHS.second;
+ return LHS.first < RHS.first;
+ }
+ };
+
+ using SortedCallTargetSet = std::set<CallTarget, CallTargetComparator>;
+ using CallTargetMap = StringMap<uint64_t>;
SampleRecord() = default;
/// Increment the number of samples for this record by \p S.
@@ -179,6 +332,18 @@
uint64_t getSamples() const { return NumSamples; }
const CallTargetMap &getCallTargets() const { return CallTargets; }
+ const SortedCallTargetSet getSortedCallTargets() const {
+ return SortCallTargets(CallTargets);
+ }
+
+ /// Sort call targets in descending order of call frequency.
+ static const SortedCallTargetSet SortCallTargets(const CallTargetMap &Targets) {
+ SortedCallTargetSet SortedTargets;
+ for (const auto &I : Targets) {
+ SortedTargets.emplace(I.first(), I.second);
+ }
+ return SortedTargets;
+ }
/// Merge the samples in \p Other into this record.
/// Optionally scale sample counts by \p Weight.
@@ -200,12 +365,136 @@
raw_ostream &operator<<(raw_ostream &OS, const SampleRecord &Sample);
+// State of context associated with FunctionSamples
+enum ContextStateMask {
+ UnknownContext = 0x0, // Profile without context
+ RawContext = 0x1, // Full context profile from input profile
+ SyntheticContext = 0x2, // Synthetic context created for context promotion
+ InlinedContext = 0x4, // Profile for context that is inlined into caller
+ MergedContext = 0x8 // Profile for context merged into base profile
+};
+
+// Sample context for FunctionSamples. It consists of the calling context,
+// the function name and context state. Internally sample context is represented
+// using StringRef, which is also the input for constructing a `SampleContext`.
+// It can accept and represent both full context string as well as context-less
+// function name.
+// Example of full context string (note the wrapping `[]`):
+// `[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]`
+// Example of context-less function name (same as AutoFDO):
+// `_Z8funcLeafi`
+class SampleContext {
+public:
+ SampleContext() : State(UnknownContext) {}
+ SampleContext(StringRef ContextStr,
+ ContextStateMask CState = UnknownContext) {
+ setContext(ContextStr, CState);
+ }
+
+ // Promote context by removing top frames (represented by `ContextStrToRemove`).
+ // Note that with string representation of context, the promotion is effectively
+ // a substr operation with `ContextStrToRemove` removed from left.
+ void promoteOnPath(StringRef ContextStrToRemove) {
+ assert(FullContext.startswith(ContextStrToRemove));
+
+ // Remove leading context and frame separator " @ ".
+ FullContext = FullContext.substr(ContextStrToRemove.size() + 3);
+ CallingContext = CallingContext.substr(ContextStrToRemove.size() + 3);
+ }
+
+ // Split the top context frame (left-most substr) from context.
+ static std::pair<StringRef, StringRef>
+ splitContextString(StringRef ContextStr) {
+ return ContextStr.split(" @ ");
+ }
+
+ // Decode context string for a frame to get function name and location.
+ // `ContextStr` is in the form of `FuncName:StartLine.Discriminator`.
+ static void decodeContextString(StringRef ContextStr, StringRef &FName,
+ LineLocation &LineLoc) {
+ // Get function name
+ auto EntrySplit = ContextStr.split(':');
+ FName = EntrySplit.first;
+
+ LineLoc = {0, 0};
+ if (!EntrySplit.second.empty()) {
+ // Get line offset, use signed int for getAsInteger so string will
+ // be parsed as signed.
+ int LineOffset = 0;
+ auto LocSplit = EntrySplit.second.split('.');
+ LocSplit.first.getAsInteger(10, LineOffset);
+ LineLoc.LineOffset = LineOffset;
+
+ // Get discriminator
+ if (!LocSplit.second.empty())
+ LocSplit.second.getAsInteger(10, LineLoc.Discriminator);
+ }
+ }
+
+ operator StringRef() const { return FullContext; }
+ bool hasState(ContextStateMask S) { return State & (uint32_t)S; }
+ void setState(ContextStateMask S) { State |= (uint32_t)S; }
+ void clearState(ContextStateMask S) { State &= (uint32_t)~S; }
+ bool hasContext() const { return State != UnknownContext; }
+ bool isBaseContext() const { return CallingContext.empty(); }
+ StringRef getName() const { return Name; }
+ StringRef getCallingContext() const { return CallingContext; }
+ StringRef getNameWithContext() const { return FullContext; }
+
+private:
+ // Give a context string, decode and populate internal states like
+ // Function name, Calling context and context state. Example of input
+ // `ContextStr`: `[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]`
+ void setContext(StringRef ContextStr, ContextStateMask CState) {
+ assert(!ContextStr.empty());
+ // Note that `[]` wrapped input indicates a full context string, otherwise
+ // it's treated as context-less function name only.
+ bool HasContext = ContextStr.startswith("[");
+ if (!HasContext && CState == UnknownContext) {
+ State = UnknownContext;
+ Name = FullContext = ContextStr;
+ } else {
+ // Assume raw context profile if unspecified
+ if (CState == UnknownContext)
+ State = RawContext;
+ else
+ State = CState;
+
+ // Remove encapsulating '[' and ']' if any
+ if (HasContext)
+ FullContext = ContextStr.substr(1, ContextStr.size() - 2);
+ else
+ FullContext = ContextStr;
+
+ // Caller is to the left of callee in context string
+ auto NameContext = FullContext.rsplit(" @ ");
+ if (NameContext.second.empty()) {
+ Name = NameContext.first;
+ CallingContext = NameContext.second;
+ } else {
+ Name = NameContext.second;
+ CallingContext = NameContext.first;
+ }
+ }
+ }
+
+ // Full context string including calling context and leaf function name
+ StringRef FullContext;
+ // Function name for the associated sample profile
+ StringRef Name;
+ // Calling context (leaf function excluded) for the associated sample profile
+ StringRef CallingContext;
+ // State of the associated sample profile
+ uint32_t State;
+};
+
class FunctionSamples;
+class SampleProfileReaderItaniumRemapper;
using BodySampleMap = std::map<LineLocation, SampleRecord>;
// NOTE: Using a StringMap here makes parsed profiles consume around 17% more
// memory, which is *very* significant for large profiles.
-using FunctionSamplesMap = std::map<std::string, FunctionSamples>;
+using FunctionSamplesMap = std::map<std::string, FunctionSamples, std::less<>>;
using CallsiteSampleMap = std::map<LineLocation, FunctionSamplesMap>;
/// Representation of the samples collected for a function.
@@ -228,6 +517,8 @@
: sampleprof_error::success;
}
+ void setTotalSamples(uint64_t Num) { TotalSamples = Num; }
+
sampleprof_error addHeadSamples(uint64_t Num, uint64_t Weight = 1) {
bool Overflowed;
TotalHeadSamples =
@@ -256,10 +547,22 @@
ErrorOr<uint64_t> findSamplesAt(uint32_t LineOffset,
uint32_t Discriminator) const {
const auto &ret = BodySamples.find(LineLocation(LineOffset, Discriminator));
- if (ret == BodySamples.end())
+ if (ret == BodySamples.end()) {
+ // For CSSPGO, in order to conserve profile size, we no longer write out
+ // locations profile for those not hit during training, so we need to
+ // treat them as zero instead of error here.
+ if (ProfileIsCS)
+ return 0;
return std::error_code();
- else
+ // A missing counter for a probe likely means the probe was not executed.
+ // Treat it as a zero count instead of an unknown count to help edge
+ // weight inference.
+ if (FunctionSamples::ProfileIsProbeBased)
+ return 0;
+ return std::error_code();
+ } else {
return ret->second.getSamples();
+ }
}
/// Returns the call target map collected at a given location.
@@ -273,6 +576,16 @@
return ret->second.getCallTargets();
}
+ /// Returns the call target map collected at a given location specified by \p
+ /// CallSite. If the location is not found in profile, return error.
+ ErrorOr<SampleRecord::CallTargetMap>
+ findCallTargetMapAt(const LineLocation &CallSite) const {
+ const auto &Ret = BodySamples.find(CallSite);
+ if (Ret == BodySamples.end())
+ return std::error_code();
+ return Ret->second.getCallTargets();
+ }
+
/// Return the function samples at the given callsite location.
FunctionSamplesMap &functionSamplesAt(const LineLocation &Loc) {
return CallsiteSamples[Loc];
@@ -287,32 +600,15 @@
return &iter->second;
}
- /// Returns a pointer to FunctionSamples at the given callsite location \p Loc
- /// with callee \p CalleeName. If no callsite can be found, relax the
- /// restriction to return the FunctionSamples at callsite location \p Loc
- /// with the maximum total sample count.
- const FunctionSamples *findFunctionSamplesAt(const LineLocation &Loc,
- StringRef CalleeName) const {
- std::string CalleeGUID;
- CalleeName = getRepInFormat(CalleeName, Format, CalleeGUID);
-
- auto iter = CallsiteSamples.find(Loc);
- if (iter == CallsiteSamples.end())
- return nullptr;
- auto FS = iter->second.find(CalleeName);
- if (FS != iter->second.end())
- return &FS->second;
- // If we cannot find exact match of the callee name, return the FS with
- // the max total count.
- uint64_t MaxTotalSamples = 0;
- const FunctionSamples *R = nullptr;
- for (const auto &NameFS : iter->second)
- if (NameFS.second.getTotalSamples() >= MaxTotalSamples) {
- MaxTotalSamples = NameFS.second.getTotalSamples();
- R = &NameFS.second;
- }
- return R;
- }
+ /// Returns a pointer to FunctionSamples at the given callsite location
+ /// \p Loc with callee \p CalleeName. If no callsite can be found, relax
+ /// the restriction to return the FunctionSamples at callsite location
+ /// \p Loc with the maximum total sample count. If \p Remapper is not
+ /// nullptr, use \p Remapper to find FunctionSamples with equivalent name
+ /// as \p CalleeName.
+ const FunctionSamples *
+ findFunctionSamplesAt(const LineLocation &Loc, StringRef CalleeName,
+ SampleProfileReaderItaniumRemapper *Remapper) const;
bool empty() const { return TotalSamples == 0; }
@@ -329,21 +625,26 @@
/// Return the sample count of the first instruction of the function.
/// The function can be either a standalone symbol or an inlined function.
uint64_t getEntrySamples() const {
+ if (FunctionSamples::ProfileIsCS && getHeadSamples()) {
+ // For CS profile, if we already have more accurate head samples
+ // counted by branch sample from caller, use them as entry samples.
+ return getHeadSamples();
+ }
+ uint64_t Count = 0;
// Use either BodySamples or CallsiteSamples which ever has the smaller
// lineno.
if (!BodySamples.empty() &&
(CallsiteSamples.empty() ||
BodySamples.begin()->first < CallsiteSamples.begin()->first))
- return BodySamples.begin()->second.getSamples();
- if (!CallsiteSamples.empty()) {
- uint64_t T = 0;
+ Count = BodySamples.begin()->second.getSamples();
+ else if (!CallsiteSamples.empty()) {
// An indirect callsite may be promoted to several inlined direct calls.
// We need to get the sum of them.
for (const auto &N_FS : CallsiteSamples.begin()->second)
- T += N_FS.second.getEntrySamples();
- return T;
+ Count += N_FS.second.getEntrySamples();
}
- return 0;
+ // Return at least 1 if total sample is not 0.
+ return Count ? Count : TotalSamples > 0;
}
/// Return all the samples collected in the body of the function.
@@ -354,11 +655,40 @@
return CallsiteSamples;
}
+ /// Return the maximum of sample counts in a function body including functions
+ /// inlined in it.
+ uint64_t getMaxCountInside() const {
+ uint64_t MaxCount = 0;
+ for (const auto &L : getBodySamples())
+ MaxCount = std::max(MaxCount, L.second.getSamples());
+ for (const auto &C : getCallsiteSamples())
+ for (const FunctionSamplesMap::value_type &F : C.second)
+ MaxCount = std::max(MaxCount, F.second.getMaxCountInside());
+ return MaxCount;
+ }
+
/// Merge the samples in \p Other into this one.
/// Optionally scale samples by \p Weight.
sampleprof_error merge(const FunctionSamples &Other, uint64_t Weight = 1) {
sampleprof_error Result = sampleprof_error::success;
Name = Other.getName();
+ if (!GUIDToFuncNameMap)
+ GUIDToFuncNameMap = Other.GUIDToFuncNameMap;
+
+ if (FunctionHash == 0) {
+ // Set the function hash code for the target profile.
+ FunctionHash = Other.getFunctionHash();
+ } else if (FunctionHash != Other.getFunctionHash()) {
+ // The two profiles coming with different valid hash codes indicates
+ // either:
+ // 1. They are same-named static functions from different compilation
+ // units (without using -unique-internal-linkage-names), or
+ // 2. They are really the same function but from different compilations.
+ // Let's bail out in either case for now, which means one profile is
+ // dropped.
+ return sampleprof_error::hash_mismatch;
+ }
+
MergeResult(Result, addTotalSamples(Other.getTotalSamples(), Weight));
MergeResult(Result, addHeadSamples(Other.getHeadSamples(), Weight));
for (const auto &I : Other.getBodySamples()) {
@@ -383,15 +713,20 @@
uint64_t Threshold) const {
if (TotalSamples <= Threshold)
return;
- S.insert(getGUID(Name));
+ auto isDeclaration = [](const Function *F) {
+ return !F || F->isDeclaration();
+ };
+ if (isDeclaration(M->getFunction(getFuncName()))) {
+ // Add to the import list only when it's defined out of module.
+ S.insert(getGUID(Name));
+ }
// Import hot CallTargets, which may not be available in IR because full
// profile annotation cannot be done until backend compilation in ThinLTO.
for (const auto &BS : BodySamples)
for (const auto &TS : BS.second.getCallTargets())
if (TS.getValue() > Threshold) {
- const Function *Callee =
- M->getFunction(getNameInModule(TS.getKey(), M));
- if (!Callee || !Callee->getSubprogram())
+ const Function *Callee = M->getFunction(getFuncName(TS.getKey()));
+ if (isDeclaration(Callee))
S.insert(getGUID(TS.getKey()));
}
for (const auto &CS : CallsiteSamples)
@@ -405,21 +740,32 @@
/// Return the function name.
StringRef getName() const { return Name; }
- /// Return the original function name if it exists in Module \p M.
- StringRef getFuncNameInModule(const Module *M) const {
- return getNameInModule(Name, M);
+ /// Return function name with context.
+ StringRef getNameWithContext() const {
+ return FunctionSamples::ProfileIsCS ? Context.getNameWithContext() : Name;
}
+ /// Return the original function name.
+ StringRef getFuncName() const { return getFuncName(Name); }
+
+ void setFunctionHash(uint64_t Hash) { FunctionHash = Hash; }
+
+ uint64_t getFunctionHash() const { return FunctionHash; }
+
/// Return the canonical name for a function, taking into account
/// suffix elision policy attributes.
static StringRef getCanonicalFnName(const Function &F) {
- static const char *knownSuffixes[] = { ".llvm.", ".part." };
auto AttrName = "sample-profile-suffix-elision-policy";
auto Attr = F.getFnAttribute(AttrName).getValueAsString();
+ return getCanonicalFnName(F.getName(), Attr);
+ }
+
+ static StringRef getCanonicalFnName(StringRef FnName, StringRef Attr = "") {
+ static const char *knownSuffixes[] = { ".llvm.", ".part." };
if (Attr == "" || Attr == "all") {
- return F.getName().split('.').first;
+ return FnName.split('.').first;
} else if (Attr == "selected") {
- StringRef Cand(F.getName());
+ StringRef Cand(FnName);
for (const auto &Suf : knownSuffixes) {
StringRef Suffix(Suf);
auto It = Cand.rfind(Suffix);
@@ -431,35 +777,38 @@
}
return Cand;
} else if (Attr == "none") {
- return F.getName();
+ return FnName;
} else {
assert(false && "internal error: unknown suffix elision policy");
}
- return F.getName();
+ return FnName;
}
- /// Translate \p Name into its original name in Module.
- /// When the Format is not SPF_Compact_Binary, \p Name needs no translation.
- /// When the Format is SPF_Compact_Binary, \p Name in current FunctionSamples
- /// is actually GUID of the original function name. getNameInModule will
- /// translate \p Name in current FunctionSamples into its original name.
- /// If the original name doesn't exist in \p M, return empty StringRef.
- StringRef getNameInModule(StringRef Name, const Module *M) const {
- if (Format != SPF_Compact_Binary)
+ /// Translate \p Name into its original name.
+ /// When profile doesn't use MD5, \p Name needs no translation.
+ /// When profile uses MD5, \p Name in current FunctionSamples
+ /// is actually GUID of the original function name. getFuncName will
+ /// translate \p Name in current FunctionSamples into its original name
+ /// by looking up in the function map GUIDToFuncNameMap.
+ /// If the original name doesn't exist in the map, return empty StringRef.
+ StringRef getFuncName(StringRef Name) const {
+ if (!UseMD5)
return Name;
- // Expect CurrentModule to be initialized by GUIDToFuncNameMapper.
- if (M != CurrentModule)
- llvm_unreachable("Input Module should be the same as CurrentModule");
- auto iter = GUIDToFuncNameMap.find(std::stoull(Name.data()));
- if (iter == GUIDToFuncNameMap.end())
- return StringRef();
- return iter->second;
+
+ assert(GUIDToFuncNameMap && "GUIDToFuncNameMap needs to be popluated first");
+ return GUIDToFuncNameMap->lookup(std::stoull(Name.data()));
}
/// Returns the line offset to the start line of the subprogram.
/// We assume that a single function will not exceed 65535 LOC.
static unsigned getOffset(const DILocation *DIL);
+ /// Returns a unique call site identifier for a given debug location of a call
+ /// instruction. This is wrapper of two scenarios, the probe-based profile and
+ /// regular profile, to hide implementation details from the sample loader and
+ /// the context tracker.
+ static LineLocation getCallSiteIdentifier(const DILocation *DIL);
+
/// Get the FunctionSamples of the inline instance where DIL originates
/// from.
///
@@ -469,58 +818,50 @@
/// tree nodes in the profile.
///
/// \returns the FunctionSamples pointer to the inlined instance.
- const FunctionSamples *findFunctionSamples(const DILocation *DIL) const;
+ /// If \p Remapper is not nullptr, it will be used to find matching
+ /// FunctionSamples with not exactly the same but equivalent name.
+ const FunctionSamples *findFunctionSamples(
+ const DILocation *DIL,
+ SampleProfileReaderItaniumRemapper *Remapper = nullptr) const;
+
+ static bool ProfileIsProbeBased;
+
+ static bool ProfileIsCS;
+
+ SampleContext &getContext() const { return Context; }
+
+ void setContext(const SampleContext &FContext) { Context = FContext; }
static SampleProfileFormat Format;
+
+ /// Whether the profile uses MD5 to represent string.
+ static bool UseMD5;
+
/// GUIDToFuncNameMap saves the mapping from GUID to the symbol name, for
- /// all the function symbols defined or declared in CurrentModule.
- static DenseMap<uint64_t, StringRef> GUIDToFuncNameMap;
- static Module *CurrentModule;
-
- class GUIDToFuncNameMapper {
- public:
- GUIDToFuncNameMapper(Module &M) {
- if (Format != SPF_Compact_Binary)
- return;
-
- for (const auto &F : M) {
- StringRef OrigName = F.getName();
- GUIDToFuncNameMap.insert({Function::getGUID(OrigName), OrigName});
- /// Local to global var promotion used by optimization like thinlto
- /// will rename the var and add suffix like ".llvm.xxx" to the
- /// original local name. In sample profile, the suffixes of function
- /// names are all stripped. Since it is possible that the mapper is
- /// built in post-thin-link phase and var promotion has been done,
- /// we need to add the substring of function name without the suffix
- /// into the GUIDToFuncNameMap.
- StringRef CanonName = getCanonicalFnName(F);
- if (CanonName != OrigName)
- GUIDToFuncNameMap.insert({Function::getGUID(CanonName), CanonName});
- }
- CurrentModule = &M;
- }
-
- ~GUIDToFuncNameMapper() {
- if (Format != SPF_Compact_Binary)
- return;
-
- GUIDToFuncNameMap.clear();
- CurrentModule = nullptr;
- }
- };
+ /// all the function symbols defined or declared in current module.
+ DenseMap<uint64_t, StringRef> *GUIDToFuncNameMap = nullptr;
// Assume the input \p Name is a name coming from FunctionSamples itself.
- // If the format is SPF_Compact_Binary, the name is already a GUID and we
+ // If UseMD5 is true, the name is already a GUID and we
// don't want to return the GUID of GUID.
static uint64_t getGUID(StringRef Name) {
- return (Format == SPF_Compact_Binary) ? std::stoull(Name.data())
- : Function::getGUID(Name);
+ return UseMD5 ? std::stoull(Name.data()) : Function::getGUID(Name);
}
+ // Find all the names in the current FunctionSamples including names in
+ // all the inline instances and names of call targets.
+ void findAllNames(DenseSet<StringRef> &NameSet) const;
+
private:
/// Mangled name of the function.
StringRef Name;
+ /// CFG hash value for the function.
+ uint64_t FunctionHash = 0;
+
+ /// Calling context for function profile
+ mutable SampleContext Context;
+
/// Total number of samples collected inside this function.
///
/// Samples are cumulative, they include all the samples collected
@@ -583,6 +924,47 @@
SamplesWithLocList V;
};
+/// ProfileSymbolList records the list of function symbols shown up
+/// in the binary used to generate the profile. It is useful to
+/// to discriminate a function being so cold as not to shown up
+/// in the profile and a function newly added.
+class ProfileSymbolList {
+public:
+ /// copy indicates whether we need to copy the underlying memory
+ /// for the input Name.
+ void add(StringRef Name, bool copy = false) {
+ if (!copy) {
+ Syms.insert(Name);
+ return;
+ }
+ Syms.insert(Name.copy(Allocator));
+ }
+
+ bool contains(StringRef Name) { return Syms.count(Name); }
+
+ void merge(const ProfileSymbolList &List) {
+ for (auto Sym : List.Syms)
+ add(Sym, true);
+ }
+
+ unsigned size() { return Syms.size(); }
+
+ void setToCompress(bool TC) { ToCompress = TC; }
+ bool toCompress() { return ToCompress; }
+
+ std::error_code read(const uint8_t *Data, uint64_t ListSize);
+ std::error_code write(raw_ostream &OS);
+ void dump(raw_ostream &OS = dbgs()) const;
+
+private:
+ // Determine whether or not to compress the symbol list when
+ // writing it into profile. The variable is unused when the symbol
+ // list is read from an existing profile.
+ bool ToCompress = false;
+ DenseSet<StringRef> Syms;
+ BumpPtrAllocator Allocator;
+};
+
} // end namespace sampleprof
} // end namespace llvm
diff --git a/linux-x64/clang/include/llvm/ProfileData/SampleProfReader.h b/linux-x64/clang/include/llvm/ProfileData/SampleProfReader.h
index 969cdea..92fe825 100644
--- a/linux-x64/clang/include/llvm/ProfileData/SampleProfReader.h
+++ b/linux-x64/clang/include/llvm/ProfileData/SampleProfReader.h
@@ -27,8 +27,9 @@
// offsetA[.discriminator]: fnA:num_of_total_samples
// offsetA1[.discriminator]: number_of_samples [fn7:num fn8:num ... ]
// ...
+// !CFGChecksum: num
//
-// This is a nested tree in which the identation represents the nesting level
+// This is a nested tree in which the indentation represents the nesting level
// of the inline stack. There are no blank lines in the file. And the spacing
// within a single line is fixed. Additional spaces will result in an error
// while reading the file.
@@ -47,10 +48,11 @@
// in the prologue of the function (second number). This head sample
// count provides an indicator of how frequently the function is invoked.
//
-// There are two types of lines in the function body.
+// There are three types of lines in the function body.
//
// * Sampled line represents the profile information of a source location.
// * Callsite line represents the profile information of a callsite.
+// * Metadata line represents extra metadata of the function.
//
// Each sampled line may contain several items. Some are optional (marked
// below):
@@ -114,6 +116,18 @@
// total number of samples collected for the inlined instance at this
// callsite
//
+// Metadata line can occur in lines with one indent only, containing extra
+// information for the top-level function. Furthermore, metadata can only
+// occur after all the body samples and callsite samples.
+// Each metadata line may contain a particular type of metadata, marked by
+// the starting characters annotated with !. We process each metadata line
+// independently, hence each metadata line has to form an independent piece
+// of information that does not require cross-line reference.
+// We support the following types of metadata:
+//
+// a. CFG Checksum (a.k.a. function hash):
+// !CFGChecksum: 12345
+//
//
// Binary format
// -------------
@@ -208,10 +222,10 @@
#ifndef LLVM_PROFILEDATA_SAMPLEPROFREADER_H
#define LLVM_PROFILEDATA_SAMPLEPROFREADER_H
+#include "llvm/ADT/Optional.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/Twine.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/LLVMContext.h"
@@ -232,9 +246,69 @@
namespace llvm {
class raw_ostream;
+class Twine;
namespace sampleprof {
+class SampleProfileReader;
+
+/// SampleProfileReaderItaniumRemapper remaps the profile data from a
+/// sample profile data reader, by applying a provided set of equivalences
+/// between components of the symbol names in the profile.
+class SampleProfileReaderItaniumRemapper {
+public:
+ SampleProfileReaderItaniumRemapper(std::unique_ptr<MemoryBuffer> B,
+ std::unique_ptr<SymbolRemappingReader> SRR,
+ SampleProfileReader &R)
+ : Buffer(std::move(B)), Remappings(std::move(SRR)), Reader(R) {
+ assert(Remappings && "Remappings cannot be nullptr");
+ }
+
+ /// Create a remapper from the given remapping file. The remapper will
+ /// be used for profile read in by Reader.
+ static ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
+ create(const std::string Filename, SampleProfileReader &Reader,
+ LLVMContext &C);
+
+ /// Create a remapper from the given Buffer. The remapper will
+ /// be used for profile read in by Reader.
+ static ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
+ create(std::unique_ptr<MemoryBuffer> &B, SampleProfileReader &Reader,
+ LLVMContext &C);
+
+ /// Apply remappings to the profile read by Reader.
+ void applyRemapping(LLVMContext &Ctx);
+
+ bool hasApplied() { return RemappingApplied; }
+
+ /// Insert function name into remapper.
+ void insert(StringRef FunctionName) { Remappings->insert(FunctionName); }
+
+ /// Query whether there is equivalent in the remapper which has been
+ /// inserted.
+ bool exist(StringRef FunctionName) {
+ return Remappings->lookup(FunctionName);
+ }
+
+ /// Return the equivalent name in the profile for \p FunctionName if
+ /// it exists.
+ Optional<StringRef> lookUpNameInProfile(StringRef FunctionName);
+
+private:
+ // The buffer holding the content read from remapping file.
+ std::unique_ptr<MemoryBuffer> Buffer;
+ std::unique_ptr<SymbolRemappingReader> Remappings;
+ // Map remapping key to the name in the profile. By looking up the
+ // key in the remapper, a given new name can be mapped to the
+ // cannonical name using the NameMap.
+ DenseMap<SymbolRemappingReader::Key, StringRef> NameMap;
+ // The Reader the remapper is servicing.
+ SampleProfileReader &Reader;
+ // Indicate whether remapping has been applied to the profile read
+ // by Reader -- by calling applyRemapping.
+ bool RemappingApplied = false;
+};
+
/// Sample-based profile reader.
///
/// Each profile contains sample counts for all the functions
@@ -273,13 +347,23 @@
/// Read and validate the file header.
virtual std::error_code readHeader() = 0;
- /// Read sample profiles from the associated file.
- virtual std::error_code read() = 0;
+ /// The interface to read sample profiles from the associated file.
+ std::error_code read() {
+ if (std::error_code EC = readImpl())
+ return EC;
+ if (Remapper)
+ Remapper->applyRemapping(Ctx);
+ FunctionSamples::UseMD5 = useMD5();
+ return sampleprof_error::success;
+ }
+
+ /// The implementaion to read sample profiles from the associated file.
+ virtual std::error_code readImpl() = 0;
/// Print the profile for \p FName on stream \p OS.
void dumpFunctionProfile(StringRef FName, raw_ostream &OS = dbgs());
- virtual void collectFuncsToUse(const Module &M) {}
+ virtual void collectFuncsFrom(const Module &M) {}
/// Print all the profiles on stream \p OS.
void dump(raw_ostream &OS = dbgs());
@@ -293,13 +377,30 @@
return getSamplesFor(CanonName);
}
+ /// Return the samples collected for function \p F, create empty
+ /// FunctionSamples if it doesn't exist.
+ FunctionSamples *getOrCreateSamplesFor(const Function &F) {
+ std::string FGUID;
+ StringRef CanonName = FunctionSamples::getCanonicalFnName(F);
+ CanonName = getRepInFormat(CanonName, useMD5(), FGUID);
+ return &Profiles[CanonName];
+ }
+
/// Return the samples collected for function \p F.
virtual FunctionSamples *getSamplesFor(StringRef Fname) {
std::string FGUID;
- Fname = getRepInFormat(Fname, getFormat(), FGUID);
+ Fname = getRepInFormat(Fname, useMD5(), FGUID);
auto It = Profiles.find(Fname);
if (It != Profiles.end())
return &It->second;
+
+ if (Remapper) {
+ if (auto NameInProfile = Remapper->lookUpNameInProfile(Fname)) {
+ auto It = Profiles.find(*NameInProfile);
+ if (It != Profiles.end())
+ return &It->second;
+ }
+ }
return nullptr;
}
@@ -307,24 +408,50 @@
StringMap<FunctionSamples> &getProfiles() { return Profiles; }
/// Report a parse error message.
- void reportError(int64_t LineNumber, Twine Msg) const {
+ void reportError(int64_t LineNumber, const Twine &Msg) const {
Ctx.diagnose(DiagnosticInfoSampleProfile(Buffer->getBufferIdentifier(),
LineNumber, Msg));
}
/// Create a sample profile reader appropriate to the file format.
+ /// Create a remapper underlying if RemapFilename is not empty.
static ErrorOr<std::unique_ptr<SampleProfileReader>>
- create(const Twine &Filename, LLVMContext &C);
+ create(const std::string Filename, LLVMContext &C,
+ const std::string RemapFilename = "");
/// Create a sample profile reader from the supplied memory buffer.
+ /// Create a remapper underlying if RemapFilename is not empty.
static ErrorOr<std::unique_ptr<SampleProfileReader>>
- create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C);
+ create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C,
+ const std::string RemapFilename = "");
/// Return the profile summary.
- ProfileSummary &getSummary() { return *(Summary.get()); }
+ ProfileSummary &getSummary() const { return *(Summary.get()); }
+
+ MemoryBuffer *getBuffer() const { return Buffer.get(); }
/// \brief Return the profile format.
- SampleProfileFormat getFormat() { return Format; }
+ SampleProfileFormat getFormat() const { return Format; }
+
+ /// Whether input profile is based on pseudo probes.
+ bool profileIsProbeBased() const { return ProfileIsProbeBased; }
+
+ /// Whether input profile is fully context-sensitive
+ bool profileIsCS() const { return ProfileIsCS; }
+
+ virtual std::unique_ptr<ProfileSymbolList> getProfileSymbolList() {
+ return nullptr;
+ };
+
+ /// It includes all the names that have samples either in outline instance
+ /// or inline instance.
+ virtual std::vector<StringRef> *getNameTable() { return nullptr; }
+ virtual bool dumpSectionInfo(raw_ostream &OS = dbgs()) { return false; };
+
+ /// Return whether names in the profile are all MD5 numbers.
+ virtual bool useMD5() { return false; }
+
+ SampleProfileReaderItaniumRemapper *getRemapper() { return Remapper.get(); }
protected:
/// Map every function to its associated profile.
@@ -352,6 +479,13 @@
/// Compute summary for this profile.
void computeSummary();
+ std::unique_ptr<SampleProfileReaderItaniumRemapper> Remapper;
+
+ /// \brief Whether samples are collected based on pseudo probes.
+ bool ProfileIsProbeBased = false;
+
+ bool ProfileIsCS = false;
+
/// \brief The format of sample.
SampleProfileFormat Format = SPF_None;
};
@@ -365,7 +499,7 @@
std::error_code readHeader() override { return sampleprof_error::success; }
/// Read sample profiles from the associated file.
- std::error_code read() override;
+ std::error_code readImpl() override;
/// Return true if \p Buffer is in the format supported by this class.
static bool hasFormat(const MemoryBuffer &Buffer);
@@ -381,7 +515,11 @@
virtual std::error_code readHeader() override;
/// Read sample profiles from the associated file.
- std::error_code read() override;
+ std::error_code readImpl() override;
+
+ /// It includes all the names that have samples either in outline instance
+ /// or inline instance.
+ virtual std::vector<StringRef> *getNameTable() override { return &NameTable; }
protected:
/// Read a numeric value of type T from the profile.
@@ -411,43 +549,160 @@
bool at_eof() const { return Data >= End; }
/// Read the next function profile instance.
- std::error_code readFuncProfile();
+ std::error_code readFuncProfile(const uint8_t *Start);
/// Read the contents of the given profile instance.
std::error_code readProfile(FunctionSamples &FProfile);
+ /// Read the contents of Magic number and Version number.
+ std::error_code readMagicIdent();
+
+ /// Read profile summary.
+ std::error_code readSummary();
+
+ /// Read the whole name table.
+ virtual std::error_code readNameTable();
+
/// Points to the current location in the buffer.
const uint8_t *Data = nullptr;
/// Points to the end of the buffer.
const uint8_t *End = nullptr;
+ /// Function name table.
+ std::vector<StringRef> NameTable;
+
+ /// Read a string indirectly via the name table.
+ virtual ErrorOr<StringRef> readStringFromTable();
+
private:
std::error_code readSummaryEntry(std::vector<ProfileSummaryEntry> &Entries);
virtual std::error_code verifySPMagic(uint64_t Magic) = 0;
-
- /// Read profile summary.
- std::error_code readSummary();
-
- /// Read the whole name table.
- virtual std::error_code readNameTable() = 0;
-
- /// Read a string indirectly via the name table.
- virtual ErrorOr<StringRef> readStringFromTable() = 0;
};
class SampleProfileReaderRawBinary : public SampleProfileReaderBinary {
private:
- /// Function name table.
- std::vector<StringRef> NameTable;
virtual std::error_code verifySPMagic(uint64_t Magic) override;
- virtual std::error_code readNameTable() override;
- /// Read a string indirectly via the name table.
- virtual ErrorOr<StringRef> readStringFromTable() override;
public:
- SampleProfileReaderRawBinary(std::unique_ptr<MemoryBuffer> B, LLVMContext &C)
- : SampleProfileReaderBinary(std::move(B), C, SPF_Binary) {}
+ SampleProfileReaderRawBinary(std::unique_ptr<MemoryBuffer> B, LLVMContext &C,
+ SampleProfileFormat Format = SPF_Binary)
+ : SampleProfileReaderBinary(std::move(B), C, Format) {}
+
+ /// \brief Return true if \p Buffer is in the format supported by this class.
+ static bool hasFormat(const MemoryBuffer &Buffer);
+};
+
+/// SampleProfileReaderExtBinaryBase/SampleProfileWriterExtBinaryBase defines
+/// the basic structure of the extensible binary format.
+/// The format is organized in sections except the magic and version number
+/// at the beginning. There is a section table before all the sections, and
+/// each entry in the table describes the entry type, start, size and
+/// attributes. The format in each section is defined by the section itself.
+///
+/// It is easy to add a new section while maintaining the backward
+/// compatibility of the profile. Nothing extra needs to be done. If we want
+/// to extend an existing section, like add cache misses information in
+/// addition to the sample count in the profile body, we can add a new section
+/// with the extension and retire the existing section, and we could choose
+/// to keep the parser of the old section if we want the reader to be able
+/// to read both new and old format profile.
+///
+/// SampleProfileReaderExtBinary/SampleProfileWriterExtBinary define the
+/// commonly used sections of a profile in extensible binary format. It is
+/// possible to define other types of profile inherited from
+/// SampleProfileReaderExtBinaryBase/SampleProfileWriterExtBinaryBase.
+class SampleProfileReaderExtBinaryBase : public SampleProfileReaderBinary {
+private:
+ std::error_code decompressSection(const uint8_t *SecStart,
+ const uint64_t SecSize,
+ const uint8_t *&DecompressBuf,
+ uint64_t &DecompressBufSize);
+
+ BumpPtrAllocator Allocator;
+
+protected:
+ std::vector<SecHdrTableEntry> SecHdrTable;
+ std::error_code readSecHdrTableEntry(uint32_t Idx);
+ std::error_code readSecHdrTable();
+
+ std::error_code readFuncMetadata();
+ std::error_code readFuncOffsetTable();
+ std::error_code readFuncProfiles();
+ std::error_code readMD5NameTable();
+ std::error_code readNameTableSec(bool IsMD5);
+ std::error_code readProfileSymbolList();
+
+ virtual std::error_code readHeader() override;
+ virtual std::error_code verifySPMagic(uint64_t Magic) override = 0;
+ virtual std::error_code readOneSection(const uint8_t *Start, uint64_t Size,
+ const SecHdrTableEntry &Entry);
+ // placeholder for subclasses to dispatch their own section readers.
+ virtual std::error_code readCustomSection(const SecHdrTableEntry &Entry) = 0;
+ virtual ErrorOr<StringRef> readStringFromTable() override;
+
+ std::unique_ptr<ProfileSymbolList> ProfSymList;
+
+ /// The table mapping from function name to the offset of its FunctionSample
+ /// towards file start.
+ DenseMap<StringRef, uint64_t> FuncOffsetTable;
+ /// The set containing the functions to use when compiling a module.
+ DenseSet<StringRef> FuncsToUse;
+ /// Use all functions from the input profile.
+ bool UseAllFuncs = true;
+
+ /// Use fixed length MD5 instead of ULEB128 encoding so NameTable doesn't
+ /// need to be read in up front and can be directly accessed using index.
+ bool FixedLengthMD5 = false;
+ /// The starting address of NameTable containing fixed length MD5.
+ const uint8_t *MD5NameMemStart = nullptr;
+
+ /// If MD5 is used in NameTable section, the section saves uint64_t data.
+ /// The uint64_t data has to be converted to a string and then the string
+ /// will be used to initialize StringRef in NameTable.
+ /// Note NameTable contains StringRef so it needs another buffer to own
+ /// the string data. MD5StringBuf serves as the string buffer that is
+ /// referenced by NameTable (vector of StringRef). We make sure
+ /// the lifetime of MD5StringBuf is not shorter than that of NameTable.
+ std::unique_ptr<std::vector<std::string>> MD5StringBuf;
+
+public:
+ SampleProfileReaderExtBinaryBase(std::unique_ptr<MemoryBuffer> B,
+ LLVMContext &C, SampleProfileFormat Format)
+ : SampleProfileReaderBinary(std::move(B), C, Format) {}
+
+ /// Read sample profiles in extensible format from the associated file.
+ std::error_code readImpl() override;
+
+ /// Get the total size of all \p Type sections.
+ uint64_t getSectionSize(SecType Type);
+ /// Get the total size of header and all sections.
+ uint64_t getFileSize();
+ virtual bool dumpSectionInfo(raw_ostream &OS = dbgs()) override;
+
+ /// Collect functions with definitions in Module \p M.
+ void collectFuncsFrom(const Module &M) override;
+
+ /// Return whether names in the profile are all MD5 numbers.
+ virtual bool useMD5() override { return MD5StringBuf.get(); }
+
+ virtual std::unique_ptr<ProfileSymbolList> getProfileSymbolList() override {
+ return std::move(ProfSymList);
+ };
+};
+
+class SampleProfileReaderExtBinary : public SampleProfileReaderExtBinaryBase {
+private:
+ virtual std::error_code verifySPMagic(uint64_t Magic) override;
+ virtual std::error_code
+ readCustomSection(const SecHdrTableEntry &Entry) override {
+ return sampleprof_error::success;
+ };
+
+public:
+ SampleProfileReaderExtBinary(std::unique_ptr<MemoryBuffer> B, LLVMContext &C,
+ SampleProfileFormat Format = SPF_Ext_Binary)
+ : SampleProfileReaderExtBinaryBase(std::move(B), C, Format) {}
/// \brief Return true if \p Buffer is in the format supported by this class.
static bool hasFormat(const MemoryBuffer &Buffer);
@@ -462,6 +717,8 @@
DenseMap<StringRef, uint64_t> FuncOffsetTable;
/// The set containing the functions to use when compiling a module.
DenseSet<StringRef> FuncsToUse;
+ /// Use all functions from the input profile.
+ bool UseAllFuncs = true;
virtual std::error_code verifySPMagic(uint64_t Magic) override;
virtual std::error_code readNameTable() override;
/// Read a string indirectly via the name table.
@@ -478,10 +735,13 @@
static bool hasFormat(const MemoryBuffer &Buffer);
/// Read samples only for functions to use.
- std::error_code read() override;
+ std::error_code readImpl() override;
/// Collect functions to be used when compiling Module \p M.
- void collectFuncsToUse(const Module &M) override;
+ void collectFuncsFrom(const Module &M) override;
+
+ /// Return whether names in the profile are all MD5 numbers.
+ virtual bool useMD5() override { return true; }
};
using InlineCallStack = SmallVector<FunctionSamples *, 10>;
@@ -509,7 +769,7 @@
std::error_code readHeader() override;
/// Read sample profiles from the associated file.
- std::error_code read() override;
+ std::error_code readImpl() override;
/// Return true if \p Buffer is in the format supported by this class.
static bool hasFormat(const MemoryBuffer &Buffer);
@@ -537,44 +797,6 @@
static const uint32_t GCOVTagAFDOFunction = 0xac000000;
};
-/// A profile data reader proxy that remaps the profile data from another
-/// sample profile data reader, by applying a provided set of equivalences
-/// between components of the symbol names in the profile.
-class SampleProfileReaderItaniumRemapper : public SampleProfileReader {
-public:
- SampleProfileReaderItaniumRemapper(
- std::unique_ptr<MemoryBuffer> B, LLVMContext &C,
- std::unique_ptr<SampleProfileReader> Underlying)
- : SampleProfileReader(std::move(B), C, Underlying->getFormat()) {
- Profiles = std::move(Underlying->getProfiles());
- Summary = takeSummary(*Underlying);
- // Keep the underlying reader alive; the profile data may contain
- // StringRefs referencing names in its name table.
- UnderlyingReader = std::move(Underlying);
- }
-
- /// Create a remapped sample profile from the given remapping file and
- /// underlying samples.
- static ErrorOr<std::unique_ptr<SampleProfileReader>>
- create(const Twine &Filename, LLVMContext &C,
- std::unique_ptr<SampleProfileReader> Underlying);
-
- /// Read and validate the file header.
- std::error_code readHeader() override { return sampleprof_error::success; }
-
- /// Read remapping file and apply it to the sample profile.
- std::error_code read() override;
-
- /// Return the samples collected for function \p F.
- FunctionSamples *getSamplesFor(StringRef FunctionName) override;
- using SampleProfileReader::getSamplesFor;
-
-private:
- SymbolRemappingReader Remappings;
- DenseMap<SymbolRemappingReader::Key, FunctionSamples*> SampleMap;
- std::unique_ptr<SampleProfileReader> UnderlyingReader;
-};
-
} // end namespace sampleprof
} // end namespace llvm
diff --git a/linux-x64/clang/include/llvm/ProfileData/SampleProfWriter.h b/linux-x64/clang/include/llvm/ProfileData/SampleProfWriter.h
index 81e6e3a..fc568f0 100644
--- a/linux-x64/clang/include/llvm/ProfileData/SampleProfWriter.h
+++ b/linux-x64/clang/include/llvm/ProfileData/SampleProfWriter.h
@@ -36,7 +36,7 @@
/// Write sample profiles in \p S.
///
/// \returns status code of the file update operation.
- virtual std::error_code write(const FunctionSamples &S) = 0;
+ virtual std::error_code writeSample(const FunctionSamples &S) = 0;
/// Write all the sample profiles in the given map of samples.
///
@@ -56,6 +56,11 @@
static ErrorOr<std::unique_ptr<SampleProfileWriter>>
create(std::unique_ptr<raw_ostream> &OS, SampleProfileFormat Format);
+ virtual void setProfileSymbolList(ProfileSymbolList *PSL) {}
+ virtual void setToCompressAllSections() {}
+ virtual void setUseMD5() {}
+ virtual void setPartialProfile() {}
+
protected:
SampleProfileWriter(std::unique_ptr<raw_ostream> &OS)
: OutputStream(std::move(OS)) {}
@@ -64,6 +69,10 @@
virtual std::error_code
writeHeader(const StringMap<FunctionSamples> &ProfileMap) = 0;
+ // Write function profiles to the profile file.
+ virtual std::error_code
+ writeFuncProfiles(const StringMap<FunctionSamples> &ProfileMap);
+
/// Output stream where to emit the profile to.
std::unique_ptr<raw_ostream> OutputStream;
@@ -72,12 +81,15 @@
/// Compute summary for this profile.
void computeSummary(const StringMap<FunctionSamples> &ProfileMap);
+
+ /// Profile format.
+ SampleProfileFormat Format = SPF_None;
};
/// Sample-based profile writer (text format).
class SampleProfileWriterText : public SampleProfileWriter {
public:
- std::error_code write(const FunctionSamples &S) override;
+ std::error_code writeSample(const FunctionSamples &S) override;
protected:
SampleProfileWriterText(std::unique_ptr<raw_ostream> &OS)
@@ -102,13 +114,14 @@
/// Sample-based profile writer (binary format).
class SampleProfileWriterBinary : public SampleProfileWriter {
public:
- virtual std::error_code write(const FunctionSamples &S) override;
SampleProfileWriterBinary(std::unique_ptr<raw_ostream> &OS)
: SampleProfileWriter(OS) {}
+ virtual std::error_code writeSample(const FunctionSamples &S) override;
+
protected:
- virtual std::error_code writeNameTable() = 0;
- virtual std::error_code writeMagicIdent() = 0;
+ virtual std::error_code writeMagicIdent(SampleProfileFormat Format);
+ virtual std::error_code writeNameTable();
virtual std::error_code
writeHeader(const StringMap<FunctionSamples> &ProfileMap) override;
std::error_code writeSummary();
@@ -118,10 +131,10 @@
MapVector<StringRef, uint32_t> NameTable;
-private:
void addName(StringRef FName);
void addNames(const FunctionSamples &S);
+private:
friend ErrorOr<std::unique_ptr<SampleProfileWriter>>
SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
SampleProfileFormat Format);
@@ -129,10 +142,152 @@
class SampleProfileWriterRawBinary : public SampleProfileWriterBinary {
using SampleProfileWriterBinary::SampleProfileWriterBinary;
+};
+
+class SampleProfileWriterExtBinaryBase : public SampleProfileWriterBinary {
+ using SampleProfileWriterBinary::SampleProfileWriterBinary;
+public:
+ virtual std::error_code
+ write(const StringMap<FunctionSamples> &ProfileMap) override;
+
+ virtual void setToCompressAllSections() override;
+ void setToCompressSection(SecType Type);
+ virtual std::error_code writeSample(const FunctionSamples &S) override;
+
+ // Set to use MD5 to represent string in NameTable.
+ virtual void setUseMD5() override {
+ UseMD5 = true;
+ addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagMD5Name);
+ // MD5 will be stored as plain uint64_t instead of variable-length
+ // quantity format in NameTable section.
+ addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagFixedLengthMD5);
+ }
+
+ // Set the profile to be partial. It means the profile is for
+ // common/shared code. The common profile is usually merged from
+ // profiles collected from running other targets.
+ virtual void setPartialProfile() override {
+ addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagPartial);
+ }
+
+ virtual void setProfileSymbolList(ProfileSymbolList *PSL) override {
+ ProfSymList = PSL;
+ };
protected:
+ uint64_t markSectionStart(SecType Type, uint32_t LayoutIdx);
+ std::error_code addNewSection(SecType Sec, uint32_t LayoutIdx,
+ uint64_t SectionStart);
+ template <class SecFlagType>
+ void addSectionFlag(SecType Type, SecFlagType Flag) {
+ for (auto &Entry : SectionHdrLayout) {
+ if (Entry.Type == Type)
+ addSecFlag(Entry, Flag);
+ }
+ }
+
+ // placeholder for subclasses to dispatch their own section writers.
+ virtual std::error_code writeCustomSection(SecType Type) = 0;
+
+ virtual void initSectionHdrLayout() = 0;
+ // specify the order to write sections.
+ virtual std::error_code
+ writeSections(const StringMap<FunctionSamples> &ProfileMap) = 0;
+
+ // Dispatch section writer for each section. \p LayoutIdx is the sequence
+ // number indicating where the section is located in SectionHdrLayout.
+ virtual std::error_code
+ writeOneSection(SecType Type, uint32_t LayoutIdx,
+ const StringMap<FunctionSamples> &ProfileMap);
+
+ // Helper function to write name table.
virtual std::error_code writeNameTable() override;
- virtual std::error_code writeMagicIdent() override;
+
+ std::error_code writeFuncMetadata(const StringMap<FunctionSamples> &Profiles);
+
+ // Functions to write various kinds of sections.
+ std::error_code
+ writeNameTableSection(const StringMap<FunctionSamples> &ProfileMap);
+ std::error_code writeFuncOffsetTable();
+ std::error_code writeProfileSymbolListSection();
+
+ // Specifiy the order of sections in section header table. Note
+ // the order of sections in SecHdrTable may be different that the
+ // order in SectionHdrLayout. sample Reader will follow the order
+ // in SectionHdrLayout to read each section.
+ SmallVector<SecHdrTableEntry, 8> SectionHdrLayout;
+
+ // Save the start of SecLBRProfile so we can compute the offset to the
+ // start of SecLBRProfile for each Function's Profile and will keep it
+ // in FuncOffsetTable.
+ uint64_t SecLBRProfileStart = 0;
+
+private:
+ void allocSecHdrTable();
+ std::error_code writeSecHdrTable();
+ virtual std::error_code
+ writeHeader(const StringMap<FunctionSamples> &ProfileMap) override;
+ std::error_code compressAndOutput();
+
+ // We will swap the raw_ostream held by LocalBufStream and that
+ // held by OutputStream if we try to add a section which needs
+ // compression. After the swap, all the data written to output
+ // will be temporarily buffered into the underlying raw_string_ostream
+ // originally held by LocalBufStream. After the data writing for the
+ // section is completed, compress the data in the local buffer,
+ // swap the raw_ostream back and write the compressed data to the
+ // real output.
+ std::unique_ptr<raw_ostream> LocalBufStream;
+ // The location where the output stream starts.
+ uint64_t FileStart;
+ // The location in the output stream where the SecHdrTable should be
+ // written to.
+ uint64_t SecHdrTableOffset;
+ // The table contains SecHdrTableEntry entries in order of how they are
+ // populated in the writer. It may be different from the order in
+ // SectionHdrLayout which specifies the sequence in which sections will
+ // be read.
+ std::vector<SecHdrTableEntry> SecHdrTable;
+
+ // FuncOffsetTable maps function name to its profile offset in SecLBRProfile
+ // section. It is used to load function profile on demand.
+ MapVector<StringRef, uint64_t> FuncOffsetTable;
+ // Whether to use MD5 to represent string.
+ bool UseMD5 = false;
+
+ ProfileSymbolList *ProfSymList = nullptr;
+};
+
+class SampleProfileWriterExtBinary : public SampleProfileWriterExtBinaryBase {
+public:
+ SampleProfileWriterExtBinary(std::unique_ptr<raw_ostream> &OS)
+ : SampleProfileWriterExtBinaryBase(OS) {
+ initSectionHdrLayout();
+ }
+
+private:
+ virtual void initSectionHdrLayout() override {
+ // Note that SecFuncOffsetTable section is written after SecLBRProfile
+ // in the profile, but is put before SecLBRProfile in SectionHdrLayout.
+ //
+ // This is because sample reader follows the order of SectionHdrLayout to
+ // read each section, to read function profiles on demand sample reader
+ // need to get the offset of each function profile first.
+ //
+ // SecFuncOffsetTable section is written after SecLBRProfile in the
+ // profile because FuncOffsetTable needs to be populated while section
+ // SecLBRProfile is written.
+ SectionHdrLayout = {
+ {SecProfSummary, 0, 0, 0, 0}, {SecNameTable, 0, 0, 0, 0},
+ {SecFuncOffsetTable, 0, 0, 0, 0}, {SecLBRProfile, 0, 0, 0, 0},
+ {SecProfileSymbolList, 0, 0, 0, 0}, {SecFuncMetadata, 0, 0, 0, 0}};
+ };
+ virtual std::error_code
+ writeSections(const StringMap<FunctionSamples> &ProfileMap) override;
+
+ virtual std::error_code writeCustomSection(SecType Type) override {
+ return sampleprof_error::success;
+ };
};
// CompactBinary is a compact format of binary profile which both reduces
@@ -169,7 +324,7 @@
using SampleProfileWriterBinary::SampleProfileWriterBinary;
public:
- virtual std::error_code write(const FunctionSamples &S) override;
+ virtual std::error_code writeSample(const FunctionSamples &S) override;
virtual std::error_code
write(const StringMap<FunctionSamples> &ProfileMap) override;
@@ -181,7 +336,6 @@
/// towards profile start.
uint64_t TableOffset;
virtual std::error_code writeNameTable() override;
- virtual std::error_code writeMagicIdent() override;
virtual std::error_code
writeHeader(const StringMap<FunctionSamples> &ProfileMap) override;
std::error_code writeFuncOffsetTable();