Update prebuilt Clang to r416183b from Android.
https://android.googlesource.com/platform/prebuilts/clang/host/
linux-x86/+/06a71ddac05c22edb2d10b590e1769b3f8619bef
clang 12.0.5 (based on r416183b) from build 7284624.
Change-Id: I277a316abcf47307562d8b748b84870f31a72866
Signed-off-by: Olivier Deprez <olivier.deprez@arm.com>
diff --git a/linux-x64/clang/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/linux-x64/clang/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h
index a7ed537..78e3cee 100644
--- a/linux-x64/clang/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h
+++ b/linux-x64/clang/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h
@@ -15,9 +15,9 @@
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/Twine.h"
#include "llvm/ExecutionEngine/JITSymbol.h"
#include "llvm/ExecutionEngine/Orc/Core.h"
+#include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/Memory.h"
#include "llvm/Support/Process.h"
@@ -26,6 +26,7 @@
#include <cassert>
#include <cstdint>
#include <functional>
+#include <future>
#include <map>
#include <memory>
#include <system_error>
@@ -42,6 +43,7 @@
class Module;
class PointerType;
class Triple;
+class Twine;
class Value;
namespace orc {
@@ -53,41 +55,19 @@
/// are used by various ORC APIs to support lazy compilation
class TrampolinePool {
public:
- virtual ~TrampolinePool() {}
+ using NotifyLandingResolvedFunction =
+ unique_function<void(JITTargetAddress) const>;
+
+ using ResolveLandingFunction = unique_function<void(
+ JITTargetAddress TrampolineAddr,
+ NotifyLandingResolvedFunction OnLandingResolved) const>;
+
+ virtual ~TrampolinePool();
/// Get an available trampoline address.
/// Returns an error if no trampoline can be created.
- virtual Expected<JITTargetAddress> getTrampoline() = 0;
-
-private:
- virtual void anchor();
-};
-
-/// A trampoline pool for trampolines within the current process.
-template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool {
-public:
- using GetTrampolineLandingFunction =
- std::function<JITTargetAddress(JITTargetAddress TrampolineAddr)>;
-
- /// Creates a LocalTrampolinePool with the given RunCallback function.
- /// Returns an error if this function is unable to correctly allocate, write
- /// and protect the resolver code block.
- static Expected<std::unique_ptr<LocalTrampolinePool>>
- Create(GetTrampolineLandingFunction GetTrampolineLanding) {
- Error Err = Error::success();
-
- auto LTP = std::unique_ptr<LocalTrampolinePool>(
- new LocalTrampolinePool(std::move(GetTrampolineLanding), Err));
-
- if (Err)
- return std::move(Err);
- return std::move(LTP);
- }
-
- /// Get a free trampoline. Returns an error if one can not be provide (e.g.
- /// because the pool is empty and can not be grown).
- Expected<JITTargetAddress> getTrampoline() override {
- std::lock_guard<std::mutex> Lock(LTPMutex);
+ Expected<JITTargetAddress> getTrampoline() {
+ std::lock_guard<std::mutex> Lock(TPMutex);
if (AvailableTrampolines.empty()) {
if (auto Err = grow())
return std::move(Err);
@@ -100,21 +80,52 @@
/// Returns the given trampoline to the pool for re-use.
void releaseTrampoline(JITTargetAddress TrampolineAddr) {
- std::lock_guard<std::mutex> Lock(LTPMutex);
+ std::lock_guard<std::mutex> Lock(TPMutex);
AvailableTrampolines.push_back(TrampolineAddr);
}
+protected:
+ virtual Error grow() = 0;
+
+ std::mutex TPMutex;
+ std::vector<JITTargetAddress> AvailableTrampolines;
+};
+
+/// A trampoline pool for trampolines within the current process.
+template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool {
+public:
+ /// Creates a LocalTrampolinePool with the given RunCallback function.
+ /// Returns an error if this function is unable to correctly allocate, write
+ /// and protect the resolver code block.
+ static Expected<std::unique_ptr<LocalTrampolinePool>>
+ Create(ResolveLandingFunction ResolveLanding) {
+ Error Err = Error::success();
+
+ auto LTP = std::unique_ptr<LocalTrampolinePool>(
+ new LocalTrampolinePool(std::move(ResolveLanding), Err));
+
+ if (Err)
+ return std::move(Err);
+ return std::move(LTP);
+ }
+
private:
static JITTargetAddress reenter(void *TrampolinePoolPtr, void *TrampolineId) {
LocalTrampolinePool<ORCABI> *TrampolinePool =
static_cast<LocalTrampolinePool *>(TrampolinePoolPtr);
- return TrampolinePool->GetTrampolineLanding(static_cast<JITTargetAddress>(
- reinterpret_cast<uintptr_t>(TrampolineId)));
+
+ std::promise<JITTargetAddress> LandingAddressP;
+ auto LandingAddressF = LandingAddressP.get_future();
+
+ TrampolinePool->ResolveLanding(pointerToJITTargetAddress(TrampolineId),
+ [&](JITTargetAddress LandingAddress) {
+ LandingAddressP.set_value(LandingAddress);
+ });
+ return LandingAddressF.get();
}
- LocalTrampolinePool(GetTrampolineLandingFunction GetTrampolineLanding,
- Error &Err)
- : GetTrampolineLanding(std::move(GetTrampolineLanding)) {
+ LocalTrampolinePool(ResolveLandingFunction ResolveLanding, Error &Err)
+ : ResolveLanding(std::move(ResolveLanding)) {
ErrorAsOutParameter _(&Err);
@@ -128,8 +139,10 @@
return;
}
- ORCABI::writeResolverCode(static_cast<uint8_t *>(ResolverBlock.base()),
- &reenter, this);
+ ORCABI::writeResolverCode(static_cast<char *>(ResolverBlock.base()),
+ pointerToJITTargetAddress(ResolverBlock.base()),
+ pointerToJITTargetAddress(&reenter),
+ pointerToJITTargetAddress(this));
EC = sys::Memory::protectMappedMemory(ResolverBlock.getMemoryBlock(),
sys::Memory::MF_READ |
@@ -140,8 +153,8 @@
}
}
- Error grow() {
- assert(this->AvailableTrampolines.empty() && "Growing prematurely?");
+ Error grow() override {
+ assert(AvailableTrampolines.empty() && "Growing prematurely?");
std::error_code EC;
auto TrampolineBlock =
@@ -155,14 +168,14 @@
(sys::Process::getPageSizeEstimate() - ORCABI::PointerSize) /
ORCABI::TrampolineSize;
- uint8_t *TrampolineMem = static_cast<uint8_t *>(TrampolineBlock.base());
- ORCABI::writeTrampolines(TrampolineMem, ResolverBlock.base(),
- NumTrampolines);
+ char *TrampolineMem = static_cast<char *>(TrampolineBlock.base());
+ ORCABI::writeTrampolines(
+ TrampolineMem, pointerToJITTargetAddress(TrampolineMem),
+ pointerToJITTargetAddress(ResolverBlock.base()), NumTrampolines);
for (unsigned I = 0; I < NumTrampolines; ++I)
- this->AvailableTrampolines.push_back(
- static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(
- TrampolineMem + (I * ORCABI::TrampolineSize))));
+ AvailableTrampolines.push_back(pointerToJITTargetAddress(
+ TrampolineMem + (I * ORCABI::TrampolineSize)));
if (auto EC = sys::Memory::protectMappedMemory(
TrampolineBlock.getMemoryBlock(),
@@ -173,12 +186,10 @@
return Error::success();
}
- GetTrampolineLandingFunction GetTrampolineLanding;
+ ResolveLandingFunction ResolveLanding;
- std::mutex LTPMutex;
sys::OwningMemoryBlock ResolverBlock;
std::vector<sys::OwningMemoryBlock> TrampolineBlocks;
- std::vector<JITTargetAddress> AvailableTrampolines;
};
/// Target-independent base class for compile callback management.
@@ -201,7 +212,7 @@
ExecutionSession &ES,
JITTargetAddress ErrorHandlerAddress)
: TP(std::move(TP)), ES(ES),
- CallbacksJD(ES.createJITDylib("<Callbacks>")),
+ CallbacksJD(ES.createBareJITDylib("<Callbacks>")),
ErrorHandlerAddress(ErrorHandlerAddress) {}
void setTrampolinePool(std::unique_ptr<TrampolinePool> TP) {
@@ -241,10 +252,14 @@
JITTargetAddress ErrorHandlerAddress,
Error &Err)
: JITCompileCallbackManager(nullptr, ES, ErrorHandlerAddress) {
+ using NotifyLandingResolvedFunction =
+ TrampolinePool::NotifyLandingResolvedFunction;
+
ErrorAsOutParameter _(&Err);
auto TP = LocalTrampolinePool<ORCABI>::Create(
- [this](JITTargetAddress TrampolineAddr) {
- return executeCompileCallback(TrampolineAddr);
+ [this](JITTargetAddress TrampolineAddr,
+ NotifyLandingResolvedFunction NotifyLandingResolved) {
+ NotifyLandingResolved(executeCompileCallback(TrampolineAddr));
});
if (!TP) {
@@ -287,6 +302,61 @@
virtual void anchor();
};
+template <typename ORCABI> class LocalIndirectStubsInfo {
+public:
+ LocalIndirectStubsInfo(unsigned NumStubs, sys::OwningMemoryBlock StubsMem)
+ : NumStubs(NumStubs), StubsMem(std::move(StubsMem)) {}
+
+ static Expected<LocalIndirectStubsInfo> create(unsigned MinStubs,
+ unsigned PageSize) {
+ auto ISAS = getIndirectStubsBlockSizes<ORCABI>(MinStubs, PageSize);
+
+ assert((ISAS.StubBytes % PageSize == 0) &&
+ "StubBytes is not a page size multiple");
+ uint64_t PointerAlloc = alignTo(ISAS.PointerBytes, PageSize);
+
+ // Allocate memory for stubs and pointers in one call.
+ std::error_code EC;
+ auto StubsAndPtrsMem =
+ sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(
+ ISAS.StubBytes + PointerAlloc, nullptr,
+ sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));
+ if (EC)
+ return errorCodeToError(EC);
+
+ sys::MemoryBlock StubsBlock(StubsAndPtrsMem.base(), ISAS.StubBytes);
+ auto StubsBlockMem = static_cast<char *>(StubsAndPtrsMem.base());
+ auto PtrBlockAddress =
+ pointerToJITTargetAddress(StubsBlockMem) + ISAS.StubBytes;
+
+ ORCABI::writeIndirectStubsBlock(StubsBlockMem,
+ pointerToJITTargetAddress(StubsBlockMem),
+ PtrBlockAddress, ISAS.NumStubs);
+
+ if (auto EC = sys::Memory::protectMappedMemory(
+ StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC))
+ return errorCodeToError(EC);
+
+ return LocalIndirectStubsInfo(ISAS.NumStubs, std::move(StubsAndPtrsMem));
+ }
+
+ unsigned getNumStubs() const { return NumStubs; }
+
+ void *getStub(unsigned Idx) const {
+ return static_cast<char *>(StubsMem.base()) + Idx * ORCABI::StubSize;
+ }
+
+ void **getPtr(unsigned Idx) const {
+ char *PtrsBase =
+ static_cast<char *>(StubsMem.base()) + NumStubs * ORCABI::StubSize;
+ return reinterpret_cast<void **>(PtrsBase) + Idx;
+ }
+
+private:
+ unsigned NumStubs = 0;
+ sys::OwningMemoryBlock StubsMem;
+};
+
/// IndirectStubsManager implementation for the host architecture, e.g.
/// OrcX86_64. (See OrcArchitectureSupport.h).
template <typename TargetT>
@@ -364,13 +434,13 @@
unsigned NewStubsRequired = NumStubs - FreeStubs.size();
unsigned NewBlockId = IndirectStubsInfos.size();
- typename TargetT::IndirectStubsInfo ISI;
- if (auto Err =
- TargetT::emitIndirectStubsBlock(ISI, NewStubsRequired, nullptr))
- return Err;
- for (unsigned I = 0; I < ISI.getNumStubs(); ++I)
+ auto ISI =
+ LocalIndirectStubsInfo<TargetT>::create(NewStubsRequired, PageSize);
+ if (!ISI)
+ return ISI.takeError();
+ for (unsigned I = 0; I < ISI->getNumStubs(); ++I)
FreeStubs.push_back(std::make_pair(NewBlockId, I));
- IndirectStubsInfos.push_back(std::move(ISI));
+ IndirectStubsInfos.push_back(std::move(*ISI));
return Error::success();
}
@@ -379,12 +449,13 @@
auto Key = FreeStubs.back();
FreeStubs.pop_back();
*IndirectStubsInfos[Key.first].getPtr(Key.second) =
- reinterpret_cast<void *>(static_cast<uintptr_t>(InitAddr));
+ jitTargetAddressToPointer<void *>(InitAddr);
StubIndexes[StubName] = std::make_pair(Key, StubFlags);
}
+ unsigned PageSize = sys::Process::getPageSizeEstimate();
std::mutex StubsMutex;
- std::vector<typename TargetT::IndirectStubsInfo> IndirectStubsInfos;
+ std::vector<LocalIndirectStubsInfo<TargetT>> IndirectStubsInfos;
using StubKey = std::pair<uint16_t, uint16_t>;
std::vector<StubKey> FreeStubs;
StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes;