Update prebuilt Clang to r416183b from Android.
https://android.googlesource.com/platform/prebuilts/clang/host/
linux-x86/+/06a71ddac05c22edb2d10b590e1769b3f8619bef
clang 12.0.5 (based on r416183b) from build 7284624.
Change-Id: I277a316abcf47307562d8b748b84870f31a72866
Signed-off-by: Olivier Deprez <olivier.deprez@arm.com>
diff --git a/linux-x64/clang/include/llvm/ProfileData/SampleProf.h b/linux-x64/clang/include/llvm/ProfileData/SampleProf.h
index 7fbc857..c423466 100644
--- a/linux-x64/clang/include/llvm/ProfileData/SampleProf.h
+++ b/linux-x64/clang/include/llvm/ProfileData/SampleProf.h
@@ -18,23 +18,25 @@
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSet.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/Module.h"
+#include "llvm/Support/Allocator.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <cstdint>
#include <map>
+#include <set>
#include <string>
#include <system_error>
#include <utility>
namespace llvm {
-class raw_ostream;
-
const std::error_category &sampleprof_category();
enum class sampleprof_error {
@@ -49,7 +51,11 @@
truncated_name_table,
not_implemented,
counter_overflow,
- ostream_seek_unsupported
+ ostream_seek_unsupported,
+ compress_failed,
+ uncompress_failed,
+ zlib_unavailable,
+ hash_mismatch
};
inline std::error_code make_error_code(sampleprof_error E) {
@@ -83,6 +89,7 @@
SPF_Text = 0x1,
SPF_Compact_Binary = 0x2,
SPF_GCC = 0x3,
+ SPF_Ext_Binary = 0x4,
SPF_Binary = 0xff
};
@@ -93,18 +100,146 @@
uint64_t('2') << (64 - 56) | uint64_t(Format);
}
-// Get the proper representation of a string in the input Format.
-static inline StringRef getRepInFormat(StringRef Name,
- SampleProfileFormat Format,
+/// Get the proper representation of a string according to whether the
+/// current Format uses MD5 to represent the string.
+static inline StringRef getRepInFormat(StringRef Name, bool UseMD5,
std::string &GUIDBuf) {
if (Name.empty())
return Name;
GUIDBuf = std::to_string(Function::getGUID(Name));
- return (Format == SPF_Compact_Binary) ? StringRef(GUIDBuf) : Name;
+ return UseMD5 ? StringRef(GUIDBuf) : Name;
}
static inline uint64_t SPVersion() { return 103; }
+// Section Type used by SampleProfileExtBinaryBaseReader and
+// SampleProfileExtBinaryBaseWriter. Never change the existing
+// value of enum. Only append new ones.
+enum SecType {
+ SecInValid = 0,
+ SecProfSummary = 1,
+ SecNameTable = 2,
+ SecProfileSymbolList = 3,
+ SecFuncOffsetTable = 4,
+ SecFuncMetadata = 5,
+ // marker for the first type of profile.
+ SecFuncProfileFirst = 32,
+ SecLBRProfile = SecFuncProfileFirst
+};
+
+static inline std::string getSecName(SecType Type) {
+ switch (Type) {
+ case SecInValid:
+ return "InvalidSection";
+ case SecProfSummary:
+ return "ProfileSummarySection";
+ case SecNameTable:
+ return "NameTableSection";
+ case SecProfileSymbolList:
+ return "ProfileSymbolListSection";
+ case SecFuncOffsetTable:
+ return "FuncOffsetTableSection";
+ case SecFuncMetadata:
+ return "FunctionMetadata";
+ case SecLBRProfile:
+ return "LBRProfileSection";
+ }
+ llvm_unreachable("A SecType has no name for output");
+}
+
+// Entry type of section header table used by SampleProfileExtBinaryBaseReader
+// and SampleProfileExtBinaryBaseWriter.
+struct SecHdrTableEntry {
+ SecType Type;
+ uint64_t Flags;
+ uint64_t Offset;
+ uint64_t Size;
+ // The index indicating the location of the current entry in
+ // SectionHdrLayout table.
+ uint32_t LayoutIndex;
+};
+
+// Flags common for all sections are defined here. In SecHdrTableEntry::Flags,
+// common flags will be saved in the lower 32bits and section specific flags
+// will be saved in the higher 32 bits.
+enum class SecCommonFlags : uint32_t {
+ SecFlagInValid = 0,
+ SecFlagCompress = (1 << 0)
+};
+
+// Section specific flags are defined here.
+// !!!Note: Everytime a new enum class is created here, please add
+// a new check in verifySecFlag.
+enum class SecNameTableFlags : uint32_t {
+ SecFlagInValid = 0,
+ SecFlagMD5Name = (1 << 0),
+ // Store MD5 in fixed length instead of ULEB128 so NameTable can be
+ // accessed like an array.
+ SecFlagFixedLengthMD5 = (1 << 1)
+};
+enum class SecProfSummaryFlags : uint32_t {
+ SecFlagInValid = 0,
+ /// SecFlagPartial means the profile is for common/shared code.
+ /// The common profile is usually merged from profiles collected
+ /// from running other targets.
+ SecFlagPartial = (1 << 0)
+};
+
+enum class SecFuncMetadataFlags : uint32_t {
+ SecFlagInvalid = 0,
+ SecFlagIsProbeBased = (1 << 0),
+};
+
+// Verify section specific flag is used for the correct section.
+template <class SecFlagType>
+static inline void verifySecFlag(SecType Type, SecFlagType Flag) {
+ // No verification is needed for common flags.
+ if (std::is_same<SecCommonFlags, SecFlagType>())
+ return;
+
+ // Verification starts here for section specific flag.
+ bool IsFlagLegal = false;
+ switch (Type) {
+ case SecNameTable:
+ IsFlagLegal = std::is_same<SecNameTableFlags, SecFlagType>();
+ break;
+ case SecProfSummary:
+ IsFlagLegal = std::is_same<SecProfSummaryFlags, SecFlagType>();
+ break;
+ case SecFuncMetadata:
+ IsFlagLegal = std::is_same<SecFuncMetadataFlags, SecFlagType>();
+ break;
+ default:
+ break;
+ }
+ if (!IsFlagLegal)
+ llvm_unreachable("Misuse of a flag in an incompatible section");
+}
+
+template <class SecFlagType>
+static inline void addSecFlag(SecHdrTableEntry &Entry, SecFlagType Flag) {
+ verifySecFlag(Entry.Type, Flag);
+ auto FVal = static_cast<uint64_t>(Flag);
+ bool IsCommon = std::is_same<SecCommonFlags, SecFlagType>();
+ Entry.Flags |= IsCommon ? FVal : (FVal << 32);
+}
+
+template <class SecFlagType>
+static inline void removeSecFlag(SecHdrTableEntry &Entry, SecFlagType Flag) {
+ verifySecFlag(Entry.Type, Flag);
+ auto FVal = static_cast<uint64_t>(Flag);
+ bool IsCommon = std::is_same<SecCommonFlags, SecFlagType>();
+ Entry.Flags &= ~(IsCommon ? FVal : (FVal << 32));
+}
+
+template <class SecFlagType>
+static inline bool hasSecFlag(const SecHdrTableEntry &Entry, SecFlagType Flag) {
+ verifySecFlag(Entry.Type, Flag);
+ auto FVal = static_cast<uint64_t>(Flag);
+ bool IsCommon = std::is_same<SecCommonFlags, SecFlagType>();
+ return Entry.Flags & (IsCommon ? FVal : (FVal << 32));
+}
+
/// Represents the relative location of an instruction.
///
/// Instruction locations are specified by the line offset from the
@@ -125,6 +260,14 @@
(LineOffset == O.LineOffset && Discriminator < O.Discriminator);
}
+ bool operator==(const LineLocation &O) const {
+ return LineOffset == O.LineOffset && Discriminator == O.Discriminator;
+ }
+
+ bool operator!=(const LineLocation &O) const {
+ return LineOffset != O.LineOffset || Discriminator != O.Discriminator;
+ }
+
uint32_t LineOffset;
uint32_t Discriminator;
};
@@ -143,8 +286,18 @@
/// will be a list of one or more functions.
class SampleRecord {
public:
- using CallTargetMap = StringMap<uint64_t>;
+ using CallTarget = std::pair<StringRef, uint64_t>;
+ struct CallTargetComparator {
+ bool operator()(const CallTarget &LHS, const CallTarget &RHS) const {
+ if (LHS.second != RHS.second)
+ return LHS.second > RHS.second;
+ return LHS.first < RHS.first;
+ }
+ };
+
+ using SortedCallTargetSet = std::set<CallTarget, CallTargetComparator>;
+ using CallTargetMap = StringMap<uint64_t>;
SampleRecord() = default;
/// Increment the number of samples for this record by \p S.
@@ -179,6 +332,18 @@
uint64_t getSamples() const { return NumSamples; }
const CallTargetMap &getCallTargets() const { return CallTargets; }
+ const SortedCallTargetSet getSortedCallTargets() const {
+ return SortCallTargets(CallTargets);
+ }
+
+ /// Sort call targets in descending order of call frequency.
+ static const SortedCallTargetSet SortCallTargets(const CallTargetMap &Targets) {
+ SortedCallTargetSet SortedTargets;
+ for (const auto &I : Targets) {
+ SortedTargets.emplace(I.first(), I.second);
+ }
+ return SortedTargets;
+ }
/// Merge the samples in \p Other into this record.
/// Optionally scale sample counts by \p Weight.
@@ -200,12 +365,136 @@
raw_ostream &operator<<(raw_ostream &OS, const SampleRecord &Sample);
+// State of context associated with FunctionSamples
+enum ContextStateMask {
+ UnknownContext = 0x0, // Profile without context
+ RawContext = 0x1, // Full context profile from input profile
+ SyntheticContext = 0x2, // Synthetic context created for context promotion
+ InlinedContext = 0x4, // Profile for context that is inlined into caller
+ MergedContext = 0x8 // Profile for context merged into base profile
+};
+
+// Sample context for FunctionSamples. It consists of the calling context,
+// the function name and context state. Internally sample context is represented
+// using StringRef, which is also the input for constructing a `SampleContext`.
+// It can accept and represent both full context string as well as context-less
+// function name.
+// Example of full context string (note the wrapping `[]`):
+// `[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]`
+// Example of context-less function name (same as AutoFDO):
+// `_Z8funcLeafi`
+class SampleContext {
+public:
+ SampleContext() : State(UnknownContext) {}
+ SampleContext(StringRef ContextStr,
+ ContextStateMask CState = UnknownContext) {
+ setContext(ContextStr, CState);
+ }
+
+ // Promote context by removing top frames (represented by `ContextStrToRemove`).
+ // Note that with string representation of context, the promotion is effectively
+ // a substr operation with `ContextStrToRemove` removed from left.
+ void promoteOnPath(StringRef ContextStrToRemove) {
+ assert(FullContext.startswith(ContextStrToRemove));
+
+ // Remove leading context and frame separator " @ ".
+ FullContext = FullContext.substr(ContextStrToRemove.size() + 3);
+ CallingContext = CallingContext.substr(ContextStrToRemove.size() + 3);
+ }
+
+ // Split the top context frame (left-most substr) from context.
+ static std::pair<StringRef, StringRef>
+ splitContextString(StringRef ContextStr) {
+ return ContextStr.split(" @ ");
+ }
+
+ // Decode context string for a frame to get function name and location.
+ // `ContextStr` is in the form of `FuncName:StartLine.Discriminator`.
+ static void decodeContextString(StringRef ContextStr, StringRef &FName,
+ LineLocation &LineLoc) {
+ // Get function name
+ auto EntrySplit = ContextStr.split(':');
+ FName = EntrySplit.first;
+
+ LineLoc = {0, 0};
+ if (!EntrySplit.second.empty()) {
+ // Get line offset, use signed int for getAsInteger so string will
+ // be parsed as signed.
+ int LineOffset = 0;
+ auto LocSplit = EntrySplit.second.split('.');
+ LocSplit.first.getAsInteger(10, LineOffset);
+ LineLoc.LineOffset = LineOffset;
+
+ // Get discriminator
+ if (!LocSplit.second.empty())
+ LocSplit.second.getAsInteger(10, LineLoc.Discriminator);
+ }
+ }
+
+ operator StringRef() const { return FullContext; }
+ bool hasState(ContextStateMask S) { return State & (uint32_t)S; }
+ void setState(ContextStateMask S) { State |= (uint32_t)S; }
+ void clearState(ContextStateMask S) { State &= (uint32_t)~S; }
+ bool hasContext() const { return State != UnknownContext; }
+ bool isBaseContext() const { return CallingContext.empty(); }
+ StringRef getName() const { return Name; }
+ StringRef getCallingContext() const { return CallingContext; }
+ StringRef getNameWithContext() const { return FullContext; }
+
+private:
+ // Give a context string, decode and populate internal states like
+ // Function name, Calling context and context state. Example of input
+ // `ContextStr`: `[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]`
+ void setContext(StringRef ContextStr, ContextStateMask CState) {
+ assert(!ContextStr.empty());
+ // Note that `[]` wrapped input indicates a full context string, otherwise
+ // it's treated as context-less function name only.
+ bool HasContext = ContextStr.startswith("[");
+ if (!HasContext && CState == UnknownContext) {
+ State = UnknownContext;
+ Name = FullContext = ContextStr;
+ } else {
+ // Assume raw context profile if unspecified
+ if (CState == UnknownContext)
+ State = RawContext;
+ else
+ State = CState;
+
+ // Remove encapsulating '[' and ']' if any
+ if (HasContext)
+ FullContext = ContextStr.substr(1, ContextStr.size() - 2);
+ else
+ FullContext = ContextStr;
+
+ // Caller is to the left of callee in context string
+ auto NameContext = FullContext.rsplit(" @ ");
+ if (NameContext.second.empty()) {
+ Name = NameContext.first;
+ CallingContext = NameContext.second;
+ } else {
+ Name = NameContext.second;
+ CallingContext = NameContext.first;
+ }
+ }
+ }
+
+ // Full context string including calling context and leaf function name
+ StringRef FullContext;
+ // Function name for the associated sample profile
+ StringRef Name;
+ // Calling context (leaf function excluded) for the associated sample profile
+ StringRef CallingContext;
+ // State of the associated sample profile
+ uint32_t State;
+};
+
class FunctionSamples;
+class SampleProfileReaderItaniumRemapper;
using BodySampleMap = std::map<LineLocation, SampleRecord>;
// NOTE: Using a StringMap here makes parsed profiles consume around 17% more
// memory, which is *very* significant for large profiles.
-using FunctionSamplesMap = std::map<std::string, FunctionSamples>;
+using FunctionSamplesMap = std::map<std::string, FunctionSamples, std::less<>>;
using CallsiteSampleMap = std::map<LineLocation, FunctionSamplesMap>;
/// Representation of the samples collected for a function.
@@ -228,6 +517,8 @@
: sampleprof_error::success;
}
+ void setTotalSamples(uint64_t Num) { TotalSamples = Num; }
+
sampleprof_error addHeadSamples(uint64_t Num, uint64_t Weight = 1) {
bool Overflowed;
TotalHeadSamples =
@@ -256,10 +547,22 @@
ErrorOr<uint64_t> findSamplesAt(uint32_t LineOffset,
uint32_t Discriminator) const {
const auto &ret = BodySamples.find(LineLocation(LineOffset, Discriminator));
- if (ret == BodySamples.end())
+ if (ret == BodySamples.end()) {
+ // For CSSPGO, in order to conserve profile size, we no longer write out
+ // locations profile for those not hit during training, so we need to
+ // treat them as zero instead of error here.
+ if (ProfileIsCS)
+ return 0;
return std::error_code();
- else
+ // A missing counter for a probe likely means the probe was not executed.
+ // Treat it as a zero count instead of an unknown count to help edge
+ // weight inference.
+ if (FunctionSamples::ProfileIsProbeBased)
+ return 0;
+ return std::error_code();
+ } else {
return ret->second.getSamples();
+ }
}
/// Returns the call target map collected at a given location.
@@ -273,6 +576,16 @@
return ret->second.getCallTargets();
}
+ /// Returns the call target map collected at a given location specified by \p
+ /// CallSite. If the location is not found in profile, return error.
+ ErrorOr<SampleRecord::CallTargetMap>
+ findCallTargetMapAt(const LineLocation &CallSite) const {
+ const auto &Ret = BodySamples.find(CallSite);
+ if (Ret == BodySamples.end())
+ return std::error_code();
+ return Ret->second.getCallTargets();
+ }
+
/// Return the function samples at the given callsite location.
FunctionSamplesMap &functionSamplesAt(const LineLocation &Loc) {
return CallsiteSamples[Loc];
@@ -287,32 +600,15 @@
return &iter->second;
}
- /// Returns a pointer to FunctionSamples at the given callsite location \p Loc
- /// with callee \p CalleeName. If no callsite can be found, relax the
- /// restriction to return the FunctionSamples at callsite location \p Loc
- /// with the maximum total sample count.
- const FunctionSamples *findFunctionSamplesAt(const LineLocation &Loc,
- StringRef CalleeName) const {
- std::string CalleeGUID;
- CalleeName = getRepInFormat(CalleeName, Format, CalleeGUID);
-
- auto iter = CallsiteSamples.find(Loc);
- if (iter == CallsiteSamples.end())
- return nullptr;
- auto FS = iter->second.find(CalleeName);
- if (FS != iter->second.end())
- return &FS->second;
- // If we cannot find exact match of the callee name, return the FS with
- // the max total count.
- uint64_t MaxTotalSamples = 0;
- const FunctionSamples *R = nullptr;
- for (const auto &NameFS : iter->second)
- if (NameFS.second.getTotalSamples() >= MaxTotalSamples) {
- MaxTotalSamples = NameFS.second.getTotalSamples();
- R = &NameFS.second;
- }
- return R;
- }
+ /// Returns a pointer to FunctionSamples at the given callsite location
+ /// \p Loc with callee \p CalleeName. If no callsite can be found, relax
+ /// the restriction to return the FunctionSamples at callsite location
+ /// \p Loc with the maximum total sample count. If \p Remapper is not
+ /// nullptr, use \p Remapper to find FunctionSamples with equivalent name
+ /// as \p CalleeName.
+ const FunctionSamples *
+ findFunctionSamplesAt(const LineLocation &Loc, StringRef CalleeName,
+ SampleProfileReaderItaniumRemapper *Remapper) const;
bool empty() const { return TotalSamples == 0; }
@@ -329,21 +625,26 @@
/// Return the sample count of the first instruction of the function.
/// The function can be either a standalone symbol or an inlined function.
uint64_t getEntrySamples() const {
+ if (FunctionSamples::ProfileIsCS && getHeadSamples()) {
+ // For CS profile, if we already have more accurate head samples
+ // counted by branch sample from caller, use them as entry samples.
+ return getHeadSamples();
+ }
+ uint64_t Count = 0;
// Use either BodySamples or CallsiteSamples which ever has the smaller
// lineno.
if (!BodySamples.empty() &&
(CallsiteSamples.empty() ||
BodySamples.begin()->first < CallsiteSamples.begin()->first))
- return BodySamples.begin()->second.getSamples();
- if (!CallsiteSamples.empty()) {
- uint64_t T = 0;
+ Count = BodySamples.begin()->second.getSamples();
+ else if (!CallsiteSamples.empty()) {
// An indirect callsite may be promoted to several inlined direct calls.
// We need to get the sum of them.
for (const auto &N_FS : CallsiteSamples.begin()->second)
- T += N_FS.second.getEntrySamples();
- return T;
+ Count += N_FS.second.getEntrySamples();
}
- return 0;
+ // Return at least 1 if total sample is not 0.
+ return Count ? Count : TotalSamples > 0;
}
/// Return all the samples collected in the body of the function.
@@ -354,11 +655,40 @@
return CallsiteSamples;
}
+ /// Return the maximum of sample counts in a function body including functions
+ /// inlined in it.
+ uint64_t getMaxCountInside() const {
+ uint64_t MaxCount = 0;
+ for (const auto &L : getBodySamples())
+ MaxCount = std::max(MaxCount, L.second.getSamples());
+ for (const auto &C : getCallsiteSamples())
+ for (const FunctionSamplesMap::value_type &F : C.second)
+ MaxCount = std::max(MaxCount, F.second.getMaxCountInside());
+ return MaxCount;
+ }
+
/// Merge the samples in \p Other into this one.
/// Optionally scale samples by \p Weight.
sampleprof_error merge(const FunctionSamples &Other, uint64_t Weight = 1) {
sampleprof_error Result = sampleprof_error::success;
Name = Other.getName();
+ if (!GUIDToFuncNameMap)
+ GUIDToFuncNameMap = Other.GUIDToFuncNameMap;
+
+ if (FunctionHash == 0) {
+ // Set the function hash code for the target profile.
+ FunctionHash = Other.getFunctionHash();
+ } else if (FunctionHash != Other.getFunctionHash()) {
+ // The two profiles coming with different valid hash codes indicates
+ // either:
+ // 1. They are same-named static functions from different compilation
+ // units (without using -unique-internal-linkage-names), or
+ // 2. They are really the same function but from different compilations.
+ // Let's bail out in either case for now, which means one profile is
+ // dropped.
+ return sampleprof_error::hash_mismatch;
+ }
+
MergeResult(Result, addTotalSamples(Other.getTotalSamples(), Weight));
MergeResult(Result, addHeadSamples(Other.getHeadSamples(), Weight));
for (const auto &I : Other.getBodySamples()) {
@@ -383,15 +713,20 @@
uint64_t Threshold) const {
if (TotalSamples <= Threshold)
return;
- S.insert(getGUID(Name));
+ auto isDeclaration = [](const Function *F) {
+ return !F || F->isDeclaration();
+ };
+ if (isDeclaration(M->getFunction(getFuncName()))) {
+ // Add to the import list only when it's defined out of module.
+ S.insert(getGUID(Name));
+ }
// Import hot CallTargets, which may not be available in IR because full
// profile annotation cannot be done until backend compilation in ThinLTO.
for (const auto &BS : BodySamples)
for (const auto &TS : BS.second.getCallTargets())
if (TS.getValue() > Threshold) {
- const Function *Callee =
- M->getFunction(getNameInModule(TS.getKey(), M));
- if (!Callee || !Callee->getSubprogram())
+ const Function *Callee = M->getFunction(getFuncName(TS.getKey()));
+ if (isDeclaration(Callee))
S.insert(getGUID(TS.getKey()));
}
for (const auto &CS : CallsiteSamples)
@@ -405,21 +740,32 @@
/// Return the function name.
StringRef getName() const { return Name; }
- /// Return the original function name if it exists in Module \p M.
- StringRef getFuncNameInModule(const Module *M) const {
- return getNameInModule(Name, M);
+ /// Return function name with context.
+ StringRef getNameWithContext() const {
+ return FunctionSamples::ProfileIsCS ? Context.getNameWithContext() : Name;
}
+ /// Return the original function name.
+ StringRef getFuncName() const { return getFuncName(Name); }
+
+ void setFunctionHash(uint64_t Hash) { FunctionHash = Hash; }
+
+ uint64_t getFunctionHash() const { return FunctionHash; }
+
/// Return the canonical name for a function, taking into account
/// suffix elision policy attributes.
static StringRef getCanonicalFnName(const Function &F) {
- static const char *knownSuffixes[] = { ".llvm.", ".part." };
auto AttrName = "sample-profile-suffix-elision-policy";
auto Attr = F.getFnAttribute(AttrName).getValueAsString();
+ return getCanonicalFnName(F.getName(), Attr);
+ }
+
+ static StringRef getCanonicalFnName(StringRef FnName, StringRef Attr = "") {
+ static const char *knownSuffixes[] = { ".llvm.", ".part." };
if (Attr == "" || Attr == "all") {
- return F.getName().split('.').first;
+ return FnName.split('.').first;
} else if (Attr == "selected") {
- StringRef Cand(F.getName());
+ StringRef Cand(FnName);
for (const auto &Suf : knownSuffixes) {
StringRef Suffix(Suf);
auto It = Cand.rfind(Suffix);
@@ -431,35 +777,38 @@
}
return Cand;
} else if (Attr == "none") {
- return F.getName();
+ return FnName;
} else {
assert(false && "internal error: unknown suffix elision policy");
}
- return F.getName();
+ return FnName;
}
- /// Translate \p Name into its original name in Module.
- /// When the Format is not SPF_Compact_Binary, \p Name needs no translation.
- /// When the Format is SPF_Compact_Binary, \p Name in current FunctionSamples
- /// is actually GUID of the original function name. getNameInModule will
- /// translate \p Name in current FunctionSamples into its original name.
- /// If the original name doesn't exist in \p M, return empty StringRef.
- StringRef getNameInModule(StringRef Name, const Module *M) const {
- if (Format != SPF_Compact_Binary)
+ /// Translate \p Name into its original name.
+ /// When profile doesn't use MD5, \p Name needs no translation.
+ /// When profile uses MD5, \p Name in current FunctionSamples
+ /// is actually GUID of the original function name. getFuncName will
+ /// translate \p Name in current FunctionSamples into its original name
+ /// by looking up in the function map GUIDToFuncNameMap.
+ /// If the original name doesn't exist in the map, return empty StringRef.
+ StringRef getFuncName(StringRef Name) const {
+ if (!UseMD5)
return Name;
- // Expect CurrentModule to be initialized by GUIDToFuncNameMapper.
- if (M != CurrentModule)
- llvm_unreachable("Input Module should be the same as CurrentModule");
- auto iter = GUIDToFuncNameMap.find(std::stoull(Name.data()));
- if (iter == GUIDToFuncNameMap.end())
- return StringRef();
- return iter->second;
+
+ assert(GUIDToFuncNameMap && "GUIDToFuncNameMap needs to be popluated first");
+ return GUIDToFuncNameMap->lookup(std::stoull(Name.data()));
}
/// Returns the line offset to the start line of the subprogram.
/// We assume that a single function will not exceed 65535 LOC.
static unsigned getOffset(const DILocation *DIL);
+ /// Returns a unique call site identifier for a given debug location of a call
+ /// instruction. This is wrapper of two scenarios, the probe-based profile and
+ /// regular profile, to hide implementation details from the sample loader and
+ /// the context tracker.
+ static LineLocation getCallSiteIdentifier(const DILocation *DIL);
+
/// Get the FunctionSamples of the inline instance where DIL originates
/// from.
///
@@ -469,58 +818,50 @@
/// tree nodes in the profile.
///
/// \returns the FunctionSamples pointer to the inlined instance.
- const FunctionSamples *findFunctionSamples(const DILocation *DIL) const;
+ /// If \p Remapper is not nullptr, it will be used to find matching
+ /// FunctionSamples with not exactly the same but equivalent name.
+ const FunctionSamples *findFunctionSamples(
+ const DILocation *DIL,
+ SampleProfileReaderItaniumRemapper *Remapper = nullptr) const;
+
+ static bool ProfileIsProbeBased;
+
+ static bool ProfileIsCS;
+
+ SampleContext &getContext() const { return Context; }
+
+ void setContext(const SampleContext &FContext) { Context = FContext; }
static SampleProfileFormat Format;
+
+ /// Whether the profile uses MD5 to represent string.
+ static bool UseMD5;
+
/// GUIDToFuncNameMap saves the mapping from GUID to the symbol name, for
- /// all the function symbols defined or declared in CurrentModule.
- static DenseMap<uint64_t, StringRef> GUIDToFuncNameMap;
- static Module *CurrentModule;
-
- class GUIDToFuncNameMapper {
- public:
- GUIDToFuncNameMapper(Module &M) {
- if (Format != SPF_Compact_Binary)
- return;
-
- for (const auto &F : M) {
- StringRef OrigName = F.getName();
- GUIDToFuncNameMap.insert({Function::getGUID(OrigName), OrigName});
- /// Local to global var promotion used by optimization like thinlto
- /// will rename the var and add suffix like ".llvm.xxx" to the
- /// original local name. In sample profile, the suffixes of function
- /// names are all stripped. Since it is possible that the mapper is
- /// built in post-thin-link phase and var promotion has been done,
- /// we need to add the substring of function name without the suffix
- /// into the GUIDToFuncNameMap.
- StringRef CanonName = getCanonicalFnName(F);
- if (CanonName != OrigName)
- GUIDToFuncNameMap.insert({Function::getGUID(CanonName), CanonName});
- }
- CurrentModule = &M;
- }
-
- ~GUIDToFuncNameMapper() {
- if (Format != SPF_Compact_Binary)
- return;
-
- GUIDToFuncNameMap.clear();
- CurrentModule = nullptr;
- }
- };
+ /// all the function symbols defined or declared in current module.
+ DenseMap<uint64_t, StringRef> *GUIDToFuncNameMap = nullptr;
// Assume the input \p Name is a name coming from FunctionSamples itself.
- // If the format is SPF_Compact_Binary, the name is already a GUID and we
+ // If UseMD5 is true, the name is already a GUID and we
// don't want to return the GUID of GUID.
static uint64_t getGUID(StringRef Name) {
- return (Format == SPF_Compact_Binary) ? std::stoull(Name.data())
- : Function::getGUID(Name);
+ return UseMD5 ? std::stoull(Name.data()) : Function::getGUID(Name);
}
+ // Find all the names in the current FunctionSamples including names in
+ // all the inline instances and names of call targets.
+ void findAllNames(DenseSet<StringRef> &NameSet) const;
+
private:
/// Mangled name of the function.
StringRef Name;
+ /// CFG hash value for the function.
+ uint64_t FunctionHash = 0;
+
+ /// Calling context for function profile
+ mutable SampleContext Context;
+
/// Total number of samples collected inside this function.
///
/// Samples are cumulative, they include all the samples collected
@@ -583,6 +924,47 @@
SamplesWithLocList V;
};
+/// ProfileSymbolList records the list of function symbols shown up
+/// in the binary used to generate the profile. It is useful to
+/// to discriminate a function being so cold as not to shown up
+/// in the profile and a function newly added.
+class ProfileSymbolList {
+public:
+ /// copy indicates whether we need to copy the underlying memory
+ /// for the input Name.
+ void add(StringRef Name, bool copy = false) {
+ if (!copy) {
+ Syms.insert(Name);
+ return;
+ }
+ Syms.insert(Name.copy(Allocator));
+ }
+
+ bool contains(StringRef Name) { return Syms.count(Name); }
+
+ void merge(const ProfileSymbolList &List) {
+ for (auto Sym : List.Syms)
+ add(Sym, true);
+ }
+
+ unsigned size() { return Syms.size(); }
+
+ void setToCompress(bool TC) { ToCompress = TC; }
+ bool toCompress() { return ToCompress; }
+
+ std::error_code read(const uint8_t *Data, uint64_t ListSize);
+ std::error_code write(raw_ostream &OS);
+ void dump(raw_ostream &OS = dbgs()) const;
+
+private:
+ // Determine whether or not to compress the symbol list when
+ // writing it into profile. The variable is unused when the symbol
+ // list is read from an existing profile.
+ bool ToCompress = false;
+ DenseSet<StringRef> Syms;
+ BumpPtrAllocator Allocator;
+};
+
} // end namespace sampleprof
} // end namespace llvm