Update prebuilt Clang to r365631c1 from Android.
The version we had was segfaulting.
Bug: 132420445
Change-Id: Icb45a6fe0b4e2166f7895e669df1157cec9fb4e0
diff --git a/linux-x64/clang/include/llvm/CodeGen/AccelTable.h b/linux-x64/clang/include/llvm/CodeGen/AccelTable.h
index 9731abd..734531a 100644
--- a/linux-x64/clang/include/llvm/CodeGen/AccelTable.h
+++ b/linux-x64/clang/include/llvm/CodeGen/AccelTable.h
@@ -326,14 +326,8 @@
void emit(AsmPrinter *Asm) const override;
-#ifndef _MSC_VER
- // The line below is rejected by older versions (TBD) of MSVC.
static constexpr Atom Atoms[] = {
Atom(dwarf::DW_ATOM_die_offset, dwarf::DW_FORM_data4)};
-#else
- // FIXME: Erase this path once the minimum MSCV version has been bumped.
- static const SmallVector<Atom, 4> Atoms;
-#endif
#ifndef NDEBUG
void print(raw_ostream &OS) const override;
@@ -351,16 +345,10 @@
void emit(AsmPrinter *Asm) const override;
-#ifndef _MSC_VER
- // The line below is rejected by older versions (TBD) of MSVC.
static constexpr Atom Atoms[] = {
Atom(dwarf::DW_ATOM_die_offset, dwarf::DW_FORM_data4),
Atom(dwarf::DW_ATOM_die_tag, dwarf::DW_FORM_data2),
Atom(dwarf::DW_ATOM_type_flags, dwarf::DW_FORM_data1)};
-#else
- // FIXME: Erase this path once the minimum MSCV version has been bumped.
- static const SmallVector<Atom, 4> Atoms;
-#endif
#ifndef NDEBUG
void print(raw_ostream &OS) const override;
@@ -375,14 +363,8 @@
void emit(AsmPrinter *Asm) const override;
-#ifndef _MSC_VER
- // The line below is rejected by older versions (TBD) of MSVC.
static constexpr Atom Atoms[] = {
Atom(dwarf::DW_ATOM_die_offset, dwarf::DW_FORM_data4)};
-#else
- // FIXME: Erase this path once the minimum MSCV version has been bumped.
- static const SmallVector<Atom, 4> Atoms;
-#endif
#ifndef NDEBUG
void print(raw_ostream &OS) const override;
@@ -406,16 +388,10 @@
void emit(AsmPrinter *Asm) const override;
-#ifndef _MSC_VER
- // The line below is rejected by older versions (TBD) of MSVC.
static constexpr Atom Atoms[] = {
Atom(dwarf::DW_ATOM_die_offset, dwarf::DW_FORM_data4),
Atom(dwarf::DW_ATOM_die_tag, dwarf::DW_FORM_data2),
Atom(5, dwarf::DW_FORM_data1), Atom(6, dwarf::DW_FORM_data4)};
-#else
- // FIXME: Erase this path once the minimum MSCV version has been bumped.
- static const SmallVector<Atom, 4> Atoms;
-#endif
#ifndef NDEBUG
void print(raw_ostream &OS) const override;
diff --git a/linux-x64/clang/include/llvm/CodeGen/Analysis.h b/linux-x64/clang/include/llvm/CodeGen/Analysis.h
index 468768d..0be0ac2 100644
--- a/linux-x64/clang/include/llvm/CodeGen/Analysis.h
+++ b/linux-x64/clang/include/llvm/CodeGen/Analysis.h
@@ -25,6 +25,7 @@
namespace llvm {
class GlobalValue;
+class LLT;
class MachineBasicBlock;
class MachineFunction;
class TargetLoweringBase;
@@ -73,6 +74,25 @@
SmallVectorImpl<uint64_t> *Offsets = nullptr,
uint64_t StartingOffset = 0);
+/// Variant of ComputeValueVTs that also produces the memory VTs.
+void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty,
+ SmallVectorImpl<EVT> &ValueVTs,
+ SmallVectorImpl<EVT> *MemVTs,
+ SmallVectorImpl<uint64_t> *Offsets = nullptr,
+ uint64_t StartingOffset = 0);
+
+/// computeValueLLTs - Given an LLVM IR type, compute a sequence of
+/// LLTs that represent all the individual underlying
+/// non-aggregate types that comprise it.
+///
+/// If Offsets is non-null, it points to a vector to be filled in
+/// with the in-memory offsets of each of the individual values.
+///
+void computeValueLLTs(const DataLayout &DL, Type &Ty,
+ SmallVectorImpl<LLT> &ValueTys,
+ SmallVectorImpl<uint64_t> *Offsets = nullptr,
+ uint64_t StartingOffset = 0);
+
/// ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
GlobalValue *ExtractTypeInfo(Value *V);
diff --git a/linux-x64/clang/include/llvm/CodeGen/AsmPrinter.h b/linux-x64/clang/include/llvm/CodeGen/AsmPrinter.h
index fb12bb2..9c2097b 100644
--- a/linux-x64/clang/include/llvm/CodeGen/AsmPrinter.h
+++ b/linux-x64/clang/include/llvm/CodeGen/AsmPrinter.h
@@ -19,6 +19,7 @@
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
+#include "llvm/CodeGen/AsmPrinterHandler.h"
#include "llvm/CodeGen/DwarfStringPoolEntry.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/IR/InlineAsm.h"
@@ -32,7 +33,6 @@
namespace llvm {
-class AsmPrinterHandler;
class BasicBlock;
class BlockAddress;
class Constant;
@@ -138,16 +138,16 @@
/// Protected struct HandlerInfo and Handlers permit target extended
/// AsmPrinter adds their own handlers.
struct HandlerInfo {
- AsmPrinterHandler *Handler;
+ std::unique_ptr<AsmPrinterHandler> Handler;
const char *TimerName;
const char *TimerDescription;
const char *TimerGroupName;
const char *TimerGroupDescription;
- HandlerInfo(AsmPrinterHandler *Handler, const char *TimerName,
- const char *TimerDescription, const char *TimerGroupName,
- const char *TimerGroupDescription)
- : Handler(Handler), TimerName(TimerName),
+ HandlerInfo(std::unique_ptr<AsmPrinterHandler> Handler,
+ const char *TimerName, const char *TimerDescription,
+ const char *TimerGroupName, const char *TimerGroupDescription)
+ : Handler(std::move(Handler)), TimerName(TimerName),
TimerDescription(TimerDescription), TimerGroupName(TimerGroupName),
TimerGroupDescription(TimerGroupDescription) {}
};
@@ -315,6 +315,8 @@
void emitStackSizeSection(const MachineFunction &MF);
+ void emitRemarksSection(Module &M);
+
enum CFIMoveType { CFI_M_None, CFI_M_EH, CFI_M_Debug };
CFIMoveType needsCFIMoves() const;
@@ -510,7 +512,7 @@
void EmitSLEB128(int64_t Value, const char *Desc = nullptr) const;
/// Emit the specified unsigned leb128 value.
- void EmitULEB128(uint64_t Value, const char *Desc = nullptr) const;
+ void EmitULEB128(uint64_t Value, const char *Desc = nullptr, unsigned PadTo = 0) const;
/// Emit a .byte 42 directive that corresponds to an encoding. If verbose
/// assembly output is enabled, we output comments describing the encoding.
@@ -588,20 +590,22 @@
virtual void PrintSpecial(const MachineInstr *MI, raw_ostream &OS,
const char *Code) const;
+ /// Print the MachineOperand as a symbol. Targets with complex handling of
+ /// symbol references should override the base implementation.
+ virtual void PrintSymbolOperand(const MachineOperand &MO, raw_ostream &OS);
+
/// Print the specified operand of MI, an INLINEASM instruction, using the
/// specified assembler variant. Targets should override this to format as
/// appropriate. This method can return true if the operand is erroneous.
virtual bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
- unsigned AsmVariant, const char *ExtraCode,
- raw_ostream &OS);
+ const char *ExtraCode, raw_ostream &OS);
/// Print the specified operand of MI, an INLINEASM instruction, using the
/// specified assembler variant as an address. Targets should override this to
/// format as appropriate. This method can return true if the operand is
/// erroneous.
virtual bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo,
- unsigned AsmVariant, const char *ExtraCode,
- raw_ostream &OS);
+ const char *ExtraCode, raw_ostream &OS);
/// Let the target do anything it needs to do before emitting inlineasm.
/// \p StartInfo - the subtarget info before parsing inline asm
@@ -616,6 +620,15 @@
virtual void emitInlineAsmEnd(const MCSubtargetInfo &StartInfo,
const MCSubtargetInfo *EndInfo) const;
+ /// This emits visibility information about symbol, if this is supported by
+ /// the target.
+ void EmitVisibility(MCSymbol *Sym, unsigned Visibility,
+ bool IsDefinition = true) const;
+
+ /// This emits linkage information about \p GVSym based on \p GV, if this is
+ /// supported by the target.
+ void EmitLinkage(const GlobalValue *GV, MCSymbol *GVSym) const;
+
private:
/// Private state for PrintSpecial()
// Assign a unique ID to this machine instruction.
@@ -646,13 +659,6 @@
// Internal Implementation Details
//===------------------------------------------------------------------===//
- /// This emits visibility information about symbol, if this is supported by
- /// the target.
- void EmitVisibility(MCSymbol *Sym, unsigned Visibility,
- bool IsDefinition = true) const;
-
- void EmitLinkage(const GlobalValue *GV, MCSymbol *GVSym) const;
-
void EmitJumpTableEntry(const MachineJumpTableInfo *MJTI,
const MachineBasicBlock *MBB, unsigned uid) const;
void EmitLLVMUsedList(const ConstantArray *InitList);
diff --git a/linux-x64/clang/include/llvm/CodeGen/BasicTTIImpl.h b/linux-x64/clang/include/llvm/CodeGen/BasicTTIImpl.h
index 1e9aeab..173be72 100644
--- a/linux-x64/clang/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/linux-x64/clang/include/llvm/CodeGen/BasicTTIImpl.h
@@ -196,11 +196,12 @@
public:
/// \name Scalar TTI Implementations
/// @{
- bool allowsMisalignedMemoryAccesses(LLVMContext &Context,
- unsigned BitWidth, unsigned AddressSpace,
- unsigned Alignment, bool *Fast) const {
+ bool allowsMisalignedMemoryAccesses(LLVMContext &Context, unsigned BitWidth,
+ unsigned AddressSpace, unsigned Alignment,
+ bool *Fast) const {
EVT E = EVT::getIntegerVT(Context, BitWidth);
- return getTLI()->allowsMisalignedMemoryAccesses(E, AddressSpace, Alignment, Fast);
+ return getTLI()->allowsMisalignedMemoryAccesses(
+ E, AddressSpace, Alignment, MachineMemOperand::MONone, Fast);
}
bool hasBranchDivergence() { return false; }
@@ -292,12 +293,12 @@
}
unsigned getIntrinsicCost(Intrinsic::ID IID, Type *RetTy,
- ArrayRef<const Value *> Arguments) {
- return BaseT::getIntrinsicCost(IID, RetTy, Arguments);
+ ArrayRef<const Value *> Arguments, const User *U) {
+ return BaseT::getIntrinsicCost(IID, RetTy, Arguments, U);
}
unsigned getIntrinsicCost(Intrinsic::ID IID, Type *RetTy,
- ArrayRef<Type *> ParamTys) {
+ ArrayRef<Type *> ParamTys, const User *U) {
if (IID == Intrinsic::cttz) {
if (getTLI()->isCheapToSpeculateCttz())
return TargetTransformInfo::TCC_Basic;
@@ -310,7 +311,7 @@
return TargetTransformInfo::TCC_Expensive;
}
- return BaseT::getIntrinsicCost(IID, RetTy, ParamTys);
+ return BaseT::getIntrinsicCost(IID, RetTy, ParamTys, U);
}
unsigned getEstimatedNumberOfCaseClusters(const SwitchInst &SI,
@@ -413,6 +414,12 @@
if (TLI->isZExtFree(OpTy, Ty))
return TargetTransformInfo::TCC_Free;
return TargetTransformInfo::TCC_Basic;
+
+ case Instruction::AddrSpaceCast:
+ if (TLI->isFreeAddrSpaceCast(OpTy->getPointerAddressSpace(),
+ Ty->getPointerAddressSpace()))
+ return TargetTransformInfo::TCC_Free;
+ return TargetTransformInfo::TCC_Basic;
}
return BaseT::getOperationCost(Opcode, Ty, OpTy);
@@ -485,6 +492,13 @@
UP.BEInsns = 2;
}
+ bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
+ AssumptionCache &AC,
+ TargetLibraryInfo *LibInfo,
+ HardwareLoopInfo &HWLoopInfo) {
+ return BaseT::isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo);
+ }
+
int getInstructionLatency(const Instruction *I) {
if (isa<LoadInst>(I))
return getST()->getSchedModel().DefaultLoadLatency;
@@ -656,7 +670,7 @@
return 0;
if (Opcode == Instruction::AddrSpaceCast &&
- TLI->isNoopAddrSpaceCast(Src->getPointerAddressSpace(),
+ TLI->isFreeAddrSpaceCast(Src->getPointerAddressSpace(),
Dst->getPointerAddressSpace()))
return 0;
@@ -1057,8 +1071,8 @@
case Intrinsic::experimental_vector_reduce_and:
case Intrinsic::experimental_vector_reduce_or:
case Intrinsic::experimental_vector_reduce_xor:
- case Intrinsic::experimental_vector_reduce_fadd:
- case Intrinsic::experimental_vector_reduce_fmul:
+ case Intrinsic::experimental_vector_reduce_v2_fadd:
+ case Intrinsic::experimental_vector_reduce_v2_fmul:
case Intrinsic::experimental_vector_reduce_smax:
case Intrinsic::experimental_vector_reduce_smin:
case Intrinsic::experimental_vector_reduce_fmax:
@@ -1248,12 +1262,16 @@
case Intrinsic::experimental_vector_reduce_xor:
return ConcreteTTI->getArithmeticReductionCost(Instruction::Xor, Tys[0],
/*IsPairwiseForm=*/false);
- case Intrinsic::experimental_vector_reduce_fadd:
- return ConcreteTTI->getArithmeticReductionCost(Instruction::FAdd, Tys[0],
- /*IsPairwiseForm=*/false);
- case Intrinsic::experimental_vector_reduce_fmul:
- return ConcreteTTI->getArithmeticReductionCost(Instruction::FMul, Tys[0],
- /*IsPairwiseForm=*/false);
+ case Intrinsic::experimental_vector_reduce_v2_fadd:
+ return ConcreteTTI->getArithmeticReductionCost(
+ Instruction::FAdd, Tys[0],
+ /*IsPairwiseForm=*/false); // FIXME: Add new flag for cost of strict
+ // reductions.
+ case Intrinsic::experimental_vector_reduce_v2_fmul:
+ return ConcreteTTI->getArithmeticReductionCost(
+ Instruction::FMul, Tys[0],
+ /*IsPairwiseForm=*/false); // FIXME: Add new flag for cost of strict
+ // reductions.
case Intrinsic::experimental_vector_reduce_smax:
case Intrinsic::experimental_vector_reduce_smin:
case Intrinsic::experimental_vector_reduce_fmax:
@@ -1306,6 +1324,30 @@
CondTy, nullptr);
return Cost;
}
+ case Intrinsic::smul_fix:
+ case Intrinsic::umul_fix: {
+ unsigned ExtSize = RetTy->getScalarSizeInBits() * 2;
+ Type *ExtTy = Type::getIntNTy(RetTy->getContext(), ExtSize);
+ if (RetVF > 1)
+ ExtTy = VectorType::get(ExtTy, RetVF);
+
+ unsigned ExtOp =
+ IID == Intrinsic::smul_fix ? Instruction::SExt : Instruction::ZExt;
+
+ unsigned Cost = 0;
+ Cost += 2 * ConcreteTTI->getCastInstrCost(ExtOp, ExtTy, RetTy);
+ Cost += ConcreteTTI->getArithmeticInstrCost(Instruction::Mul, ExtTy);
+ Cost +=
+ 2 * ConcreteTTI->getCastInstrCost(Instruction::Trunc, RetTy, ExtTy);
+ Cost += ConcreteTTI->getArithmeticInstrCost(Instruction::LShr, RetTy,
+ TTI::OK_AnyValue,
+ TTI::OK_UniformConstantValue);
+ Cost += ConcreteTTI->getArithmeticInstrCost(Instruction::Shl, RetTy,
+ TTI::OK_AnyValue,
+ TTI::OK_UniformConstantValue);
+ Cost += ConcreteTTI->getArithmeticInstrCost(Instruction::Or, RetTy);
+ return Cost;
+ }
case Intrinsic::sadd_with_overflow:
case Intrinsic::ssub_with_overflow: {
Type *SumTy = RetTy->getContainedType(0);
@@ -1346,6 +1388,36 @@
OverflowTy, nullptr);
return Cost;
}
+ case Intrinsic::smul_with_overflow:
+ case Intrinsic::umul_with_overflow: {
+ Type *MulTy = RetTy->getContainedType(0);
+ Type *OverflowTy = RetTy->getContainedType(1);
+ unsigned ExtSize = MulTy->getScalarSizeInBits() * 2;
+ Type *ExtTy = Type::getIntNTy(RetTy->getContext(), ExtSize);
+ if (MulTy->isVectorTy())
+ ExtTy = VectorType::get(ExtTy, MulTy->getVectorNumElements() );
+
+ unsigned ExtOp =
+ IID == Intrinsic::smul_fix ? Instruction::SExt : Instruction::ZExt;
+
+ unsigned Cost = 0;
+ Cost += 2 * ConcreteTTI->getCastInstrCost(ExtOp, ExtTy, MulTy);
+ Cost += ConcreteTTI->getArithmeticInstrCost(Instruction::Mul, ExtTy);
+ Cost +=
+ 2 * ConcreteTTI->getCastInstrCost(Instruction::Trunc, MulTy, ExtTy);
+ Cost += ConcreteTTI->getArithmeticInstrCost(Instruction::LShr, MulTy,
+ TTI::OK_AnyValue,
+ TTI::OK_UniformConstantValue);
+
+ if (IID == Intrinsic::smul_with_overflow)
+ Cost += ConcreteTTI->getArithmeticInstrCost(
+ Instruction::AShr, MulTy, TTI::OK_AnyValue,
+ TTI::OK_UniformConstantValue);
+
+ Cost += ConcreteTTI->getCmpSelInstrCost(BinaryOperator::ICmp, MulTy,
+ OverflowTy, nullptr);
+ return Cost;
+ }
case Intrinsic::ctpop:
ISDs.push_back(ISD::CTPOP);
// In case of legalization use TCC_Expensive. This is cheaper than a
diff --git a/linux-x64/clang/include/llvm/CodeGen/CSEConfigBase.h b/linux-x64/clang/include/llvm/CodeGen/CSEConfigBase.h
new file mode 100644
index 0000000..70b5e5c
--- /dev/null
+++ b/linux-x64/clang/include/llvm/CodeGen/CSEConfigBase.h
@@ -0,0 +1,28 @@
+//===- CSEConfigBase.h - A CSEConfig interface ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_CSECONFIG_BASE_H
+#define LLVM_CODEGEN_CSECONFIG_BASE_H
+
+namespace llvm {
+// Class representing some configuration that can be done during GlobalISel's
+// CSEInfo analysis. We define it here because TargetPassConfig can't depend on
+// the GlobalISel library, and so we use this in the interface between them
+// so that the derived classes in GISel can reference generic opcodes.
+class CSEConfigBase {
+public:
+ virtual ~CSEConfigBase() = default;
+ // Hook for defining which Generic instructions should be CSEd.
+ // GISelCSEInfo currently only calls this hook when dealing with generic
+ // opcodes.
+ virtual bool shouldCSEOpc(unsigned Opc) { return false; }
+};
+
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_CSECONFIG_BASE_H
diff --git a/linux-x64/clang/include/llvm/CodeGen/CallingConvLower.h b/linux-x64/clang/include/llvm/CodeGen/CallingConvLower.h
index 78aebbe..aa339e1 100644
--- a/linux-x64/clang/include/llvm/CodeGen/CallingConvLower.h
+++ b/linux-x64/clang/include/llvm/CodeGen/CallingConvLower.h
@@ -145,7 +145,7 @@
bool needsCustom() const { return isCustom; }
- unsigned getLocReg() const { assert(isRegLoc()); return Loc; }
+ Register getLocReg() const { assert(isRegLoc()); return Loc; }
unsigned getLocMemOffset() const { assert(isMemLoc()); return Loc; }
unsigned getExtraInfo() const { return Loc; }
MVT getLocVT() const { return LocVT; }
@@ -556,7 +556,7 @@
// Sort the locations of the arguments according to their original position.
SmallVector<CCValAssign, 16> TmpArgLocs;
- std::swap(TmpArgLocs, Locs);
+ TmpArgLocs.swap(Locs);
auto B = TmpArgLocs.begin(), E = TmpArgLocs.end();
std::merge(B, B + NumFirstPassLocs, B + NumFirstPassLocs, E,
std::back_inserter(Locs),
diff --git a/linux-x64/clang/include/llvm/CodeGen/CommandFlags.inc b/linux-x64/clang/include/llvm/CodeGen/CommandFlags.inc
index 5b9564c..cb69e9f 100644
--- a/linux-x64/clang/include/llvm/CodeGen/CommandFlags.inc
+++ b/linux-x64/clang/include/llvm/CodeGen/CommandFlags.inc
@@ -271,6 +271,11 @@
EnableAddrsig("addrsig", cl::desc("Emit an address-significance table"),
cl::init(false));
+static cl::opt<bool>
+ EnableDebugEntryValues("debug-entry-values",
+ cl::desc("Emit debug info about parameter's entry values"),
+ cl::init(false));
+
// Common utility function tightly tied to the options listed here. Initializes
// a TargetOptions object with CodeGen flags and returns it.
static TargetOptions InitTargetOptionsFromCodeGenFlags() {
@@ -300,6 +305,7 @@
Options.ExceptionModel = ExceptionModel;
Options.EmitStackSizeSection = EnableStackSizeSection;
Options.EmitAddrsig = EnableAddrsig;
+ Options.EnableDebugEntryValues = EnableDebugEntryValues;
Options.MCOptions = InitMCTargetOptionsFromFlags();
diff --git a/linux-x64/clang/include/llvm/CodeGen/DIE.h b/linux-x64/clang/include/llvm/CodeGen/DIE.h
index 9e7167f..684f9e4 100644
--- a/linux-x64/clang/include/llvm/CodeGen/DIE.h
+++ b/linux-x64/clang/include/llvm/CodeGen/DIE.h
@@ -38,6 +38,7 @@
class AsmPrinter;
class DIE;
class DIEUnit;
+class DwarfCompileUnit;
class MCExpr;
class MCSection;
class MCSymbol;
@@ -230,6 +231,25 @@
};
//===--------------------------------------------------------------------===//
+/// A BaseTypeRef DIE.
+class DIEBaseTypeRef {
+ const DwarfCompileUnit *CU;
+ const uint64_t Index;
+ static constexpr unsigned ULEB128PadSize = 4;
+
+public:
+ explicit DIEBaseTypeRef(const DwarfCompileUnit *TheCU, uint64_t Idx)
+ : CU(TheCU), Index(Idx) {}
+
+ /// EmitValue - Emit base type reference.
+ void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const;
+ /// SizeOf - Determine size of the base type reference in bytes.
+ unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const;
+
+ void print(raw_ostream &O) const;
+};
+
+//===--------------------------------------------------------------------===//
/// A simple label difference DIE.
///
class DIEDelta {
@@ -349,7 +369,7 @@
/// should be stored by reference instead of by value.
using ValTy = AlignedCharArrayUnion<DIEInteger, DIEString, DIEExpr, DIELabel,
DIEDelta *, DIEEntry, DIEBlock *,
- DIELoc *, DIELocList>;
+ DIELoc *, DIELocList, DIEBaseTypeRef *>;
static_assert(sizeof(ValTy) <= sizeof(uint64_t) ||
sizeof(ValTy) <= sizeof(void *),
@@ -501,6 +521,18 @@
}
Last = &N;
}
+
+ void push_front(Node &N) {
+ assert(N.Next.getPointer() == &N && "Expected unlinked node");
+ assert(N.Next.getInt() == true && "Expected unlinked node");
+
+ if (Last) {
+ N.Next.setPointerAndInt(Last->Next.getPointer(), false);
+ Last->Next.setPointerAndInt(&N, true);
+ } else {
+ Last = &N;
+ }
+ }
};
template <class T> class IntrusiveBackList : IntrusiveBackListBase {
@@ -508,8 +540,15 @@
using IntrusiveBackListBase::empty;
void push_back(T &N) { IntrusiveBackListBase::push_back(N); }
+ void push_front(T &N) { IntrusiveBackListBase::push_front(N); }
T &back() { return *static_cast<T *>(Last); }
const T &back() const { return *static_cast<T *>(Last); }
+ T &front() {
+ return *static_cast<T *>(Last ? Last->Next.getPointer() : nullptr);
+ }
+ const T &front() const {
+ return *static_cast<T *>(Last ? Last->Next.getPointer() : nullptr);
+ }
class const_iterator;
class iterator
@@ -759,7 +798,7 @@
///
/// \returns the DIEUnit that represents the compile or type unit that owns
/// this DIE, or NULL if this DIE hasn't been added to a unit DIE.
- const DIEUnit *getUnit() const;
+ DIEUnit *getUnit() const;
void setOffset(unsigned O) { Offset = O; }
void setSize(unsigned S) { Size = S; }
@@ -772,6 +811,13 @@
return Children.back();
}
+ DIE &addChildFront(DIE *Child) {
+ assert(!Child->getParent() && "Child should be orphaned");
+ Child->Owner = this;
+ Children.push_front(*Child);
+ return Children.front();
+ }
+
/// Find a value in the DIE with the attribute given.
///
/// Returns a default-constructed DIEValue (where \a DIEValue::getType()
diff --git a/linux-x64/clang/include/llvm/CodeGen/DIEValue.def b/linux-x64/clang/include/llvm/CodeGen/DIEValue.def
index c6c4c9a..92afeb3 100644
--- a/linux-x64/clang/include/llvm/CodeGen/DIEValue.def
+++ b/linux-x64/clang/include/llvm/CodeGen/DIEValue.def
@@ -34,6 +34,7 @@
HANDLE_DIEVALUE_SMALL(String)
HANDLE_DIEVALUE_SMALL(Expr)
HANDLE_DIEVALUE_SMALL(Label)
+HANDLE_DIEVALUE_LARGE(BaseTypeRef)
HANDLE_DIEVALUE_LARGE(Delta)
HANDLE_DIEVALUE_SMALL(Entry)
HANDLE_DIEVALUE_LARGE(Block)
diff --git a/linux-x64/clang/include/llvm/CodeGen/DbgEntityHistoryCalculator.h b/linux-x64/clang/include/llvm/CodeGen/DbgEntityHistoryCalculator.h
index b5374d8..7eec75b 100644
--- a/linux-x64/clang/include/llvm/CodeGen/DbgEntityHistoryCalculator.h
+++ b/linux-x64/clang/include/llvm/CodeGen/DbgEntityHistoryCalculator.h
@@ -10,6 +10,7 @@
#define LLVM_CODEGEN_DBGVALUEHISTORYCALCULATOR_H
#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PointerIntPair.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include <utility>
@@ -21,35 +22,76 @@
class MachineInstr;
class TargetRegisterInfo;
-// For each user variable, keep a list of instruction ranges where this variable
-// is accessible. The variables are listed in order of appearance.
+/// For each user variable, keep a list of instruction ranges where this
+/// variable is accessible. The variables are listed in order of appearance.
class DbgValueHistoryMap {
- // Each instruction range starts with a DBG_VALUE instruction, specifying the
- // location of a variable, which is assumed to be valid until the end of the
- // range. If end is not specified, location is valid until the start
- // instruction of the next instruction range, or until the end of the
- // function.
public:
- using InstrRange = std::pair<const MachineInstr *, const MachineInstr *>;
- using InstrRanges = SmallVector<InstrRange, 4>;
+ /// Index in the entry vector.
+ typedef size_t EntryIndex;
+
+ /// Special value to indicate that an entry is valid until the end of the
+ /// function.
+ static const EntryIndex NoEntry = std::numeric_limits<EntryIndex>::max();
+
+ /// Specifies a change in a variable's debug value history.
+ ///
+ /// There exist two types of entries:
+ ///
+ /// * Debug value entry:
+ ///
+ /// A new debug value becomes live. If the entry's \p EndIndex is \p NoEntry,
+ /// the value is valid until the end of the function. For other values, the
+ /// index points to the entry in the entry vector that ends this debug
+ /// value. The ending entry can either be an overlapping debug value, or
+ /// an instruction that clobbers the value.
+ ///
+ /// * Clobbering entry:
+ ///
+ /// This entry's instruction clobbers one or more preceding
+ /// register-described debug values that have their end index
+ /// set to this entry's position in the entry vector.
+ class Entry {
+ public:
+ enum EntryKind { DbgValue, Clobber };
+
+ Entry(const MachineInstr *Instr, EntryKind Kind)
+ : Instr(Instr, Kind), EndIndex(NoEntry) {}
+
+ const MachineInstr *getInstr() const { return Instr.getPointer(); }
+ EntryIndex getEndIndex() const { return EndIndex; }
+ EntryKind getEntryKind() const { return Instr.getInt(); }
+
+ bool isClobber() const { return getEntryKind() == Clobber; }
+ bool isDbgValue() const { return getEntryKind() == DbgValue; }
+ bool isClosed() const { return EndIndex != NoEntry; }
+
+ void endEntry(EntryIndex EndIndex);
+
+ private:
+ PointerIntPair<const MachineInstr *, 1, EntryKind> Instr;
+ EntryIndex EndIndex;
+ };
+ using Entries = SmallVector<Entry, 4>;
using InlinedEntity = std::pair<const DINode *, const DILocation *>;
- using InstrRangesMap = MapVector<InlinedEntity, InstrRanges>;
+ using EntriesMap = MapVector<InlinedEntity, Entries>;
private:
- InstrRangesMap VarInstrRanges;
+ EntriesMap VarEntries;
public:
- void startInstrRange(InlinedEntity Var, const MachineInstr &MI);
- void endInstrRange(InlinedEntity Var, const MachineInstr &MI);
+ bool startDbgValue(InlinedEntity Var, const MachineInstr &MI,
+ EntryIndex &NewIndex);
+ EntryIndex startClobber(InlinedEntity Var, const MachineInstr &MI);
- // Returns register currently describing @Var. If @Var is currently
- // unaccessible or is not described by a register, returns 0.
- unsigned getRegisterForVar(InlinedEntity Var) const;
+ Entry &getEntry(InlinedEntity Var, EntryIndex Index) {
+ auto &Entries = VarEntries[Var];
+ return Entries[Index];
+ }
- bool empty() const { return VarInstrRanges.empty(); }
- void clear() { VarInstrRanges.clear(); }
- InstrRangesMap::const_iterator begin() const { return VarInstrRanges.begin(); }
- InstrRangesMap::const_iterator end() const { return VarInstrRanges.end(); }
+ bool empty() const { return VarEntries.empty(); }
+ void clear() { VarEntries.clear(); }
+ EntriesMap::const_iterator begin() const { return VarEntries.begin(); }
+ EntriesMap::const_iterator end() const { return VarEntries.end(); }
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_DUMP_METHOD void dump() const;
diff --git a/linux-x64/clang/include/llvm/CodeGen/DebugHandlerBase.h b/linux-x64/clang/include/llvm/CodeGen/DebugHandlerBase.h
index 3771d48..4008d59 100644
--- a/linux-x64/clang/include/llvm/CodeGen/DebugHandlerBase.h
+++ b/linux-x64/clang/include/llvm/CodeGen/DebugHandlerBase.h
@@ -129,7 +129,7 @@
const MCExpr *getFunctionLocalOffsetAfterInsn(const MachineInstr *MI);
/// If this type is derived from a base type then return base type size.
- static uint64_t getBaseTypeSize(const DITypeRef TyRef);
+ static uint64_t getBaseTypeSize(const DIType *Ty);
};
}
diff --git a/linux-x64/clang/include/llvm/CodeGen/FastISel.h b/linux-x64/clang/include/llvm/CodeGen/FastISel.h
index 394324b..f09b59d 100644
--- a/linux-x64/clang/include/llvm/CodeGen/FastISel.h
+++ b/linux-x64/clang/include/llvm/CodeGen/FastISel.h
@@ -527,7 +527,7 @@
/// Select and emit code for a binary operator instruction, which has
/// an opcode which directly corresponds to the given ISD opcode.
bool selectBinaryOp(const User *I, unsigned ISDOpcode);
- bool selectFNeg(const User *I);
+ bool selectFNeg(const User *I, const Value *In);
bool selectGetElementPtr(const User *I);
bool selectStackmap(const CallInst *I);
bool selectPatchpoint(const CallInst *I);
diff --git a/linux-x64/clang/include/llvm/CodeGen/FunctionLoweringInfo.h b/linux-x64/clang/include/llvm/CodeGen/FunctionLoweringInfo.h
index f5f37d1..fb60191 100644
--- a/linux-x64/clang/include/llvm/CodeGen/FunctionLoweringInfo.h
+++ b/linux-x64/clang/include/llvm/CodeGen/FunctionLoweringInfo.h
@@ -13,7 +13,6 @@
#ifndef LLVM_CODEGEN_FUNCTIONLOWERINGINFO_H
#define LLVM_CODEGEN_FUNCTIONLOWERINGINFO_H
-
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/DenseMap.h"
@@ -21,6 +20,7 @@
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/LegacyDivergenceAnalysis.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
@@ -57,6 +57,7 @@
const TargetLowering *TLI;
MachineRegisterInfo *RegInfo;
BranchProbabilityInfo *BPI;
+ const LegacyDivergenceAnalysis *DA;
/// CanLowerReturn - true iff the function's return value can be lowered to
/// registers.
bool CanLowerReturn;
@@ -71,48 +72,6 @@
/// MBBMap - A mapping from LLVM basic blocks to their machine code entry.
DenseMap<const BasicBlock*, MachineBasicBlock *> MBBMap;
- /// A map from swifterror value in a basic block to the virtual register it is
- /// currently represented by.
- DenseMap<std::pair<const MachineBasicBlock *, const Value *>, unsigned>
- SwiftErrorVRegDefMap;
-
- /// A list of upward exposed vreg uses that need to be satisfied by either a
- /// copy def or a phi node at the beginning of the basic block representing
- /// the predecessor(s) swifterror value.
- DenseMap<std::pair<const MachineBasicBlock *, const Value *>, unsigned>
- SwiftErrorVRegUpwardsUse;
-
- /// A map from instructions that define/use a swifterror value to the virtual
- /// register that represents that def/use.
- llvm::DenseMap<PointerIntPair<const Instruction *, 1, bool>, unsigned>
- SwiftErrorVRegDefUses;
-
- /// The swifterror argument of the current function.
- const Value *SwiftErrorArg;
-
- using SwiftErrorValues = SmallVector<const Value*, 1>;
- /// A function can only have a single swifterror argument. And if it does
- /// have a swifterror argument, it must be the first entry in
- /// SwiftErrorVals.
- SwiftErrorValues SwiftErrorVals;
-
- /// Get or create the swifterror value virtual register in
- /// SwiftErrorVRegDefMap for this basic block.
- unsigned getOrCreateSwiftErrorVReg(const MachineBasicBlock *,
- const Value *);
-
- /// Set the swifterror virtual register in the SwiftErrorVRegDefMap for this
- /// basic block.
- void setCurrentSwiftErrorVReg(const MachineBasicBlock *MBB, const Value *,
- unsigned);
-
- /// Get or create the swifterror value virtual register for a def of a
- /// swifterror by an instruction.
- std::pair<unsigned, bool> getOrCreateSwiftErrorVRegDefAt(const Instruction *);
- std::pair<unsigned, bool>
- getOrCreateSwiftErrorVRegUseAt(const Instruction *, const MachineBasicBlock *,
- const Value *);
-
/// ValueMap - Since we emit code for the function a basic block at a time,
/// we must remember which virtual registers hold the values for
/// cross-basic-block values.
@@ -240,9 +199,11 @@
return ValueMap.count(V);
}
- unsigned CreateReg(MVT VT);
+ unsigned CreateReg(MVT VT, bool isDivergent = false);
- unsigned CreateRegs(Type *Ty);
+ unsigned CreateRegs(const Value *V);
+
+ unsigned CreateRegs(Type *Ty, bool isDivergent = false);
unsigned InitializeRegForValue(const Value *V) {
// Tokens never live in vregs.
@@ -251,7 +212,7 @@
unsigned &R = ValueMap[V];
assert(R == 0 && "Already initialized this value register!");
assert(VirtReg2Value.empty());
- return R = CreateRegs(V->getType());
+ return R = CreateRegs(V);
}
/// GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the
diff --git a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/CSEInfo.h b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/CSEInfo.h
index 97f3cd5..5a44e67 100644
--- a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/CSEInfo.h
+++ b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/CSEInfo.h
@@ -13,6 +13,7 @@
#define LLVM_CODEGEN_GLOBALISEL_CSEINFO_H
#include "llvm/ADT/FoldingSet.h"
+#include "llvm/CodeGen/CSEConfigBase.h"
#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
#include "llvm/CodeGen/GlobalISel/GISelWorkList.h"
#include "llvm/CodeGen/GlobalISel/Utils.h"
@@ -36,25 +37,27 @@
void Profile(FoldingSetNodeID &ID);
};
-// Class representing some configuration that can be done during CSE analysis.
-// Currently it only supports shouldCSE method that each pass can set.
-class CSEConfig {
+// A CSE config for fully optimized builds.
+class CSEConfigFull : public CSEConfigBase {
public:
- virtual ~CSEConfig() = default;
- // Hook for defining which Generic instructions should be CSEd.
- // GISelCSEInfo currently only calls this hook when dealing with generic
- // opcodes.
- virtual bool shouldCSEOpc(unsigned Opc);
+ virtual ~CSEConfigFull() = default;
+ virtual bool shouldCSEOpc(unsigned Opc) override;
};
-// TODO: Find a better place for this.
// Commonly used for O0 config.
-class CSEConfigConstantOnly : public CSEConfig {
+class CSEConfigConstantOnly : public CSEConfigBase {
public:
virtual ~CSEConfigConstantOnly() = default;
virtual bool shouldCSEOpc(unsigned Opc) override;
};
+// Returns the standard expected CSEConfig for the given optimization level.
+// We have this logic here so targets can make use of it from their derived
+// TargetPassConfig, but can't put this logic into TargetPassConfig directly
+// because the CodeGen library can't depend on GlobalISel.
+std::unique_ptr<CSEConfigBase>
+getStandardCSEConfigForOpt(CodeGenOpt::Level Level);
+
/// The CSE Analysis object.
/// This installs itself as a delegate to the MachineFunction to track
/// new instructions as well as deletions. It however will not be able to
@@ -73,7 +76,7 @@
FoldingSet<UniqueMachineInstr> CSEMap;
MachineRegisterInfo *MRI = nullptr;
MachineFunction *MF = nullptr;
- std::unique_ptr<CSEConfig> CSEOpt;
+ std::unique_ptr<CSEConfigBase> CSEOpt;
/// Keep a cache of UniqueInstrs for each MachineInstr. In GISel,
/// often instructions are mutated (while their ID has completely changed).
/// Whenever mutation happens, invalidate the UniqueMachineInstr for the
@@ -138,7 +141,9 @@
void releaseMemory();
- void setCSEConfig(std::unique_ptr<CSEConfig> Opt) { CSEOpt = std::move(Opt); }
+ void setCSEConfig(std::unique_ptr<CSEConfigBase> Opt) {
+ CSEOpt = std::move(Opt);
+ }
bool shouldCSE(unsigned Opc) const;
@@ -198,11 +203,12 @@
bool AlreadyComputed = false;
public:
- /// Takes a CSEConfig object that defines what opcodes get CSEd.
+ /// Takes a CSEConfigBase object that defines what opcodes get CSEd.
/// If CSEConfig is already set, and the CSE Analysis has been preserved,
/// it will not use the new CSEOpt(use Recompute to force using the new
/// CSEOpt).
- GISelCSEInfo &get(std::unique_ptr<CSEConfig> CSEOpt, bool ReCompute = false);
+ GISelCSEInfo &get(std::unique_ptr<CSEConfigBase> CSEOpt,
+ bool ReCompute = false);
void setMF(MachineFunction &MFunc) { MF = &MFunc; }
void setComputed(bool Computed) { AlreadyComputed = Computed; }
void releaseMemory() { Info.releaseMemory(); }
diff --git a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/CallLowering.h b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/CallLowering.h
index 9b72b70..d8d15bd 100644
--- a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/CallLowering.h
+++ b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/CallLowering.h
@@ -15,6 +15,7 @@
#define LLVM_CODEGEN_GLOBALISEL_CALLLOWERING_H
#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallVector.h"
#include "llvm/CodeGen/CallingConvLower.h"
#include "llvm/CodeGen/TargetCallingConv.h"
#include "llvm/IR/CallSite.h"
@@ -42,14 +43,19 @@
virtual void anchor();
public:
struct ArgInfo {
- unsigned Reg;
+ SmallVector<Register, 4> Regs;
Type *Ty;
ISD::ArgFlagsTy Flags;
bool IsFixed;
- ArgInfo(unsigned Reg, Type *Ty, ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy{},
- bool IsFixed = true)
- : Reg(Reg), Ty(Ty), Flags(Flags), IsFixed(IsFixed) {}
+ ArgInfo(ArrayRef<Register> Regs, Type *Ty,
+ ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy{}, bool IsFixed = true)
+ : Regs(Regs.begin(), Regs.end()), Ty(Ty), Flags(Flags),
+ IsFixed(IsFixed) {
+ // FIXME: We should have just one way of saying "no register".
+ assert((Ty->isVoidTy() == (Regs.empty() || Regs[0] == 0)) &&
+ "only void types should have no register");
+ }
};
/// Argument handling is mostly uniform between the four places that
@@ -65,24 +71,28 @@
virtual ~ValueHandler() = default;
+ /// Returns true if the handler is dealing with formal arguments,
+ /// not with return values etc.
+ virtual bool isArgumentHandler() const { return false; }
+
/// Materialize a VReg containing the address of the specified
/// stack-based object. This is either based on a FrameIndex or
/// direct SP manipulation, depending on the context. \p MPO
/// should be initialized to an appropriate description of the
/// address created.
- virtual unsigned getStackAddress(uint64_t Size, int64_t Offset,
+ virtual Register getStackAddress(uint64_t Size, int64_t Offset,
MachinePointerInfo &MPO) = 0;
/// The specified value has been assigned to a physical register,
/// handle the appropriate COPY (either to or from) and mark any
/// relevant uses/defines as needed.
- virtual void assignValueToReg(unsigned ValVReg, unsigned PhysReg,
+ virtual void assignValueToReg(Register ValVReg, Register PhysReg,
CCValAssign &VA) = 0;
/// The specified value has been assigned to a stack
/// location. Load or store it there, with appropriate extension
/// if necessary.
- virtual void assignValueToAddress(unsigned ValVReg, unsigned Addr,
+ virtual void assignValueToAddress(Register ValVReg, Register Addr,
uint64_t Size, MachinePointerInfo &MPO,
CCValAssign &VA) = 0;
@@ -97,7 +107,7 @@
llvm_unreachable("Custom values not supported");
}
- unsigned extendRegister(unsigned ValReg, CCValAssign &VA);
+ Register extendRegister(Register ValReg, CCValAssign &VA);
virtual bool assignArg(unsigned ValNo, MVT ValVT, MVT LocVT,
CCValAssign::LocInfo LocInfo, const ArgInfo &Info,
@@ -129,6 +139,24 @@
void setArgFlags(ArgInfo &Arg, unsigned OpIdx, const DataLayout &DL,
const FuncInfoTy &FuncInfo) const;
+ /// Generate instructions for packing \p SrcRegs into one big register
+ /// corresponding to the aggregate type \p PackedTy.
+ ///
+ /// \param SrcRegs should contain one virtual register for each base type in
+ /// \p PackedTy, as returned by computeValueLLTs.
+ ///
+ /// \return The packed register.
+ Register packRegs(ArrayRef<Register> SrcRegs, Type *PackedTy,
+ MachineIRBuilder &MIRBuilder) const;
+
+ /// Generate instructions for unpacking \p SrcReg into the \p DstRegs
+ /// corresponding to the aggregate type \p PackedTy.
+ ///
+ /// \param DstRegs should contain one virtual register for each base type in
+ /// \p PackedTy, as returned by computeValueLLTs.
+ void unpackRegs(ArrayRef<Register> DstRegs, Register SrcReg, Type *PackedTy,
+ MachineIRBuilder &MIRBuilder) const;
+
/// Invoke Handler::assignArg on each of the given \p Args and then use
/// \p Callback to move them to the assigned locations.
///
@@ -140,28 +168,51 @@
CallLowering(const TargetLowering *TLI) : TLI(TLI) {}
virtual ~CallLowering() = default;
+ /// \return true if the target is capable of handling swifterror values that
+ /// have been promoted to a specified register. The extended versions of
+ /// lowerReturn and lowerCall should be implemented.
+ virtual bool supportSwiftError() const {
+ return false;
+ }
+
/// This hook must be implemented to lower outgoing return values, described
/// by \p Val, into the specified virtual registers \p VRegs.
/// This hook is used by GlobalISel.
///
+ /// \p SwiftErrorVReg is non-zero if the function has a swifterror parameter
+ /// that needs to be implicitly returned.
+ ///
/// \return True if the lowering succeeds, false otherwise.
virtual bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val,
- ArrayRef<unsigned> VRegs) const {
+ ArrayRef<Register> VRegs,
+ Register SwiftErrorVReg) const {
+ if (!supportSwiftError()) {
+ assert(SwiftErrorVReg == 0 && "attempt to use unsupported swifterror");
+ return lowerReturn(MIRBuilder, Val, VRegs);
+ }
+ return false;
+ }
+
+ /// This hook behaves as the extended lowerReturn function, but for targets
+ /// that do not support swifterror value promotion.
+ virtual bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val,
+ ArrayRef<Register> VRegs) const {
return false;
}
/// This hook must be implemented to lower the incoming (formal)
- /// arguments, described by \p Args, for GlobalISel. Each argument
- /// must end up in the related virtual register described by VRegs.
- /// In other words, the first argument should end up in VRegs[0],
- /// the second in VRegs[1], and so on.
+ /// arguments, described by \p VRegs, for GlobalISel. Each argument
+ /// must end up in the related virtual registers described by \p VRegs.
+ /// In other words, the first argument should end up in \c VRegs[0],
+ /// the second in \c VRegs[1], and so on. For each argument, there will be one
+ /// register for each non-aggregate type, as returned by \c computeValueLLTs.
/// \p MIRBuilder is set to the proper insertion for the argument
/// lowering.
///
/// \return True if the lowering succeeded, false otherwise.
virtual bool lowerFormalArguments(MachineIRBuilder &MIRBuilder,
const Function &F,
- ArrayRef<unsigned> VRegs) const {
+ ArrayRef<ArrayRef<Register>> VRegs) const {
return false;
}
@@ -173,20 +224,31 @@
/// \p Callee is the destination of the call. It should be either a register,
/// globaladdress, or externalsymbol.
///
- /// \p ResTy is the type returned by the function
+ /// \p OrigRet is a descriptor for the return type of the function.
///
- /// \p ResReg is the generic virtual register that the returned
- /// value should be lowered into.
+ /// \p OrigArgs is a list of descriptors of the arguments passed to the
+ /// function.
///
- /// \p ArgTys is a list of the types each member of \p ArgRegs has; used by
- /// the target to decide which register/stack slot should be allocated.
- ///
- /// \p ArgRegs is a list of virtual registers containing each argument that
- /// needs to be passed.
+ /// \p SwiftErrorVReg is non-zero if the call has a swifterror inout
+ /// parameter, and contains the vreg that the swifterror should be copied into
+ /// after the call.
///
/// \return true if the lowering succeeded, false otherwise.
virtual bool lowerCall(MachineIRBuilder &MIRBuilder, CallingConv::ID CallConv,
const MachineOperand &Callee, const ArgInfo &OrigRet,
+ ArrayRef<ArgInfo> OrigArgs,
+ Register SwiftErrorVReg) const {
+ if (!supportSwiftError()) {
+ assert(SwiftErrorVReg == 0 && "trying to use unsupported swifterror");
+ return lowerCall(MIRBuilder, CallConv, Callee, OrigRet, OrigArgs);
+ }
+ return false;
+ }
+
+ /// This hook behaves as the extended lowerCall function, but for targets that
+ /// do not support swifterror value promotion.
+ virtual bool lowerCall(MachineIRBuilder &MIRBuilder, CallingConv::ID CallConv,
+ const MachineOperand &Callee, const ArgInfo &OrigRet,
ArrayRef<ArgInfo> OrigArgs) const {
return false;
}
@@ -196,11 +258,18 @@
///
/// \p CI is the call/invoke instruction.
///
- /// \p ResReg is a register where the call's return value should be stored (or
- /// 0 if there is no return value).
+ /// \p ResRegs are the registers where the call's return value should be
+ /// stored (or 0 if there is no return value). There will be one register for
+ /// each non-aggregate type, as returned by \c computeValueLLTs.
///
- /// \p ArgRegs is a list of virtual registers containing each argument that
- /// needs to be passed.
+ /// \p ArgRegs is a list of lists of virtual registers containing each
+ /// argument that needs to be passed (argument \c i should be placed in \c
+ /// ArgRegs[i]). For each argument, there will be one register for each
+ /// non-aggregate type, as returned by \c computeValueLLTs.
+ ///
+ /// \p SwiftErrorVReg is non-zero if the call has a swifterror inout
+ /// parameter, and contains the vreg that the swifterror should be copied into
+ /// after the call.
///
/// \p GetCalleeReg is a callback to materialize a register for the callee if
/// the target determines it cannot jump to the destination based purely on \p
@@ -209,7 +278,8 @@
///
/// \return true if the lowering succeeded, false otherwise.
bool lowerCall(MachineIRBuilder &MIRBuilder, ImmutableCallSite CS,
- unsigned ResReg, ArrayRef<unsigned> ArgRegs,
+ ArrayRef<Register> ResRegs,
+ ArrayRef<ArrayRef<Register>> ArgRegs, Register SwiftErrorVReg,
std::function<unsigned()> GetCalleeReg) const;
};
diff --git a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index ee30ba9..0c50c9c 100644
--- a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -17,6 +17,9 @@
#ifndef LLVM_CODEGEN_GLOBALISEL_COMBINER_HELPER_H
#define LLVM_CODEGEN_GLOBALISEL_COMBINER_HELPER_H
+#include "llvm/CodeGen/LowLevelType.h"
+#include "llvm/CodeGen/Register.h"
+
namespace llvm {
class GISelChangeObserver;
@@ -25,6 +28,12 @@
class MachineInstr;
class MachineOperand;
+struct PreferredTuple {
+ LLT Ty; // The result type of the extend.
+ unsigned ExtendOpcode; // G_ANYEXT/G_SEXT/G_ZEXT
+ MachineInstr *MI;
+};
+
class CombinerHelper {
MachineIRBuilder &Builder;
MachineRegisterInfo &MRI;
@@ -34,20 +43,27 @@
CombinerHelper(GISelChangeObserver &Observer, MachineIRBuilder &B);
/// MachineRegisterInfo::replaceRegWith() and inform the observer of the changes
- void replaceRegWith(MachineRegisterInfo &MRI, unsigned FromReg, unsigned ToReg) const;
+ void replaceRegWith(MachineRegisterInfo &MRI, Register FromReg, Register ToReg) const;
/// Replace a single register operand with a new register and inform the
/// observer of the changes.
void replaceRegOpWith(MachineRegisterInfo &MRI, MachineOperand &FromRegOp,
- unsigned ToReg) const;
+ Register ToReg) const;
/// If \p MI is COPY, try to combine it.
/// Returns true if MI changed.
bool tryCombineCopy(MachineInstr &MI);
+ bool matchCombineCopy(MachineInstr &MI);
+ void applyCombineCopy(MachineInstr &MI);
/// If \p MI is extend that consumes the result of a load, try to combine it.
/// Returns true if MI changed.
bool tryCombineExtendingLoads(MachineInstr &MI);
+ bool matchCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo);
+ void applyCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo);
+
+ bool matchCombineBr(MachineInstr &MI);
+ bool tryCombineBr(MachineInstr &MI);
/// Try to transform \p MI by using all of the above
/// combine functions. Returns true if changed.
diff --git a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/GISelWorkList.h b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/GISelWorkList.h
index 626a666..b0bb519 100644
--- a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/GISelWorkList.h
+++ b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/GISelWorkList.h
@@ -32,6 +32,10 @@
SmallVector<MachineInstr *, N> Worklist;
DenseMap<MachineInstr *, unsigned> WorklistMap;
+#ifndef NDEBUG
+ bool Finalized = true;
+#endif
+
public:
GISelWorkList() : WorklistMap(N) {}
@@ -39,16 +43,50 @@
unsigned size() const { return WorklistMap.size(); }
+ // Since we don't know ahead of time how many instructions we're going to add
+ // to the worklist, and migrating densemap's elements is quite expensive
+ // everytime we resize, only insert to the smallvector (typically during the
+ // initial phase of populating lists). Before the worklist can be used,
+ // finalize should be called. Also assert with NDEBUG if list is ever used
+ // without finalizing. Note that unlike insert, we won't check for duplicates
+ // - so the ideal place to use this is during the initial prepopulating phase
+ // of most passes.
+ void deferred_insert(MachineInstr *I) {
+ Worklist.push_back(I);
+#ifndef NDEBUG
+ Finalized = false;
+#endif
+ }
+
+ // This should only be called when using deferred_insert.
+ // This asserts that the WorklistMap is empty, and then
+ // inserts all the elements in the Worklist into the map.
+ // It also asserts if there are any duplicate elements found.
+ void finalize() {
+ assert(WorklistMap.empty() && "Expecting empty worklistmap");
+ if (Worklist.size() > N)
+ WorklistMap.reserve(Worklist.size());
+ for (unsigned i = 0; i < Worklist.size(); ++i)
+ if (!WorklistMap.try_emplace(Worklist[i], i).second)
+ llvm_unreachable("Duplicate elements in the list");
+#ifndef NDEBUG
+ Finalized = true;
+#endif
+ }
+
/// Add the specified instruction to the worklist if it isn't already in it.
void insert(MachineInstr *I) {
+ assert(Finalized && "GISelWorkList used without finalizing");
if (WorklistMap.try_emplace(I, Worklist.size()).second)
Worklist.push_back(I);
}
/// Remove I from the worklist if it exists.
void remove(const MachineInstr *I) {
+ assert((Finalized || WorklistMap.empty()) && "Neither finalized nor empty");
auto It = WorklistMap.find(I);
- if (It == WorklistMap.end()) return; // Not in worklist.
+ if (It == WorklistMap.end())
+ return; // Not in worklist.
// Don't bother moving everything down, just null out the slot.
Worklist[It->second] = nullptr;
@@ -62,6 +100,7 @@
}
MachineInstr *pop_back_val() {
+ assert(Finalized && "GISelWorkList used without finalizing");
MachineInstr *I;
do {
I = Worklist.pop_back_val();
diff --git a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/IRTranslator.h
index c75d823..8654ba8 100644
--- a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/IRTranslator.h
+++ b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/IRTranslator.h
@@ -22,7 +22,9 @@
#include "llvm/ADT/SmallVector.h"
#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
#include "llvm/CodeGen/GlobalISel/Types.h"
+#include "llvm/CodeGen/SwiftErrorValueTracking.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/SwitchLoweringUtils.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/Support/Allocator.h"
#include <memory>
@@ -36,6 +38,7 @@
class CallLowering;
class Constant;
class DataLayout;
+class FunctionLoweringInfo;
class Instruction;
class MachineBasicBlock;
class MachineFunction;
@@ -68,7 +71,7 @@
public:
ValueToVRegInfo() = default;
- using VRegListT = SmallVector<unsigned, 1>;
+ using VRegListT = SmallVector<Register, 1>;
using OffsetListT = SmallVector<uint64_t, 1>;
using const_vreg_iterator =
@@ -163,6 +166,8 @@
/// this function.
DenseMap<const AllocaInst *, int> FrameIndices;
+ SwiftErrorValueTracking SwiftError;
+
/// \name Methods for translating form LLVM IR to MachineInstr.
/// \see ::translate for general information on the translate methods.
/// @{
@@ -195,7 +200,7 @@
/// the function.
///
/// \return true if the materialization succeeded.
- bool translate(const Constant &C, unsigned Reg);
+ bool translate(const Constant &C, Register Reg);
/// Translate an LLVM bitcast into generic IR. Either a COPY or a G_BITCAST is
/// emitted.
@@ -211,7 +216,7 @@
bool translateMemfunc(const CallInst &CI, MachineIRBuilder &MIRBuilder,
unsigned ID);
- void getStackGuard(unsigned DstReg, MachineIRBuilder &MIRBuilder);
+ void getStackGuard(Register DstReg, MachineIRBuilder &MIRBuilder);
bool translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
MachineIRBuilder &MIRBuilder);
@@ -232,14 +237,6 @@
bool translateInlineAsm(const CallInst &CI, MachineIRBuilder &MIRBuilder);
- // FIXME: temporary function to expose previous interface to call lowering
- // until it is refactored.
- /// Combines all component registers of \p V into a single scalar with size
- /// "max(Offsets) + last size".
- unsigned packRegs(const Value &V, MachineIRBuilder &MIRBuilder);
-
- void unpackRegs(const Value &V, unsigned Src, MachineIRBuilder &MIRBuilder);
-
/// Returns true if the value should be split into multiple LLTs.
/// If \p Offsets is given then the split type's offsets will be stored in it.
/// If \p Offsets is not empty it will be cleared first.
@@ -290,7 +287,42 @@
/// \pre \p U is a branch instruction.
bool translateBr(const User &U, MachineIRBuilder &MIRBuilder);
+ // Begin switch lowering functions.
+ bool emitJumpTableHeader(SwitchCG::JumpTable &JT,
+ SwitchCG::JumpTableHeader &JTH,
+ MachineBasicBlock *HeaderBB);
+ void emitJumpTable(SwitchCG::JumpTable &JT, MachineBasicBlock *MBB);
+
+ void emitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB,
+ MachineIRBuilder &MIB);
+
+ bool lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W,
+ MachineBasicBlock *SwitchMBB,
+ MachineBasicBlock *CurMBB,
+ MachineBasicBlock *DefaultMBB,
+ MachineIRBuilder &MIB,
+ MachineFunction::iterator BBI,
+ BranchProbability UnhandledProbs,
+ SwitchCG::CaseClusterIt I,
+ MachineBasicBlock *Fallthrough,
+ bool FallthroughUnreachable);
+
+ bool lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I,
+ Value *Cond,
+ MachineBasicBlock *Fallthrough,
+ bool FallthroughUnreachable,
+ BranchProbability UnhandledProbs,
+ MachineBasicBlock *CurMBB,
+ MachineIRBuilder &MIB,
+ MachineBasicBlock *SwitchMBB);
+
+ bool lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W, Value *Cond,
+ MachineBasicBlock *SwitchMBB,
+ MachineBasicBlock *DefaultMBB,
+ MachineIRBuilder &MIB);
+
bool translateSwitch(const User &U, MachineIRBuilder &MIRBuilder);
+ // End switch lowering section.
bool translateIndirectBr(const User &U, MachineIRBuilder &MIRBuilder);
@@ -416,6 +448,7 @@
bool translateAtomicCmpXchg(const User &U, MachineIRBuilder &MIRBuilder);
bool translateAtomicRMW(const User &U, MachineIRBuilder &MIRBuilder);
+ bool translateFence(const User &U, MachineIRBuilder &MIRBuilder);
// Stubs to keep the compiler happy while we implement the rest of the
// translation.
@@ -431,9 +464,6 @@
bool translateCatchSwitch(const User &U, MachineIRBuilder &MIRBuilder) {
return false;
}
- bool translateFence(const User &U, MachineIRBuilder &MIRBuilder) {
- return false;
- }
bool translateAddrSpaceCast(const User &U, MachineIRBuilder &MIRBuilder) {
return translateCast(TargetOpcode::G_ADDRSPACE_CAST, U, MIRBuilder);
}
@@ -478,19 +508,50 @@
/// Current optimization remark emitter. Used to report failures.
std::unique_ptr<OptimizationRemarkEmitter> ORE;
+ FunctionLoweringInfo FuncInfo;
+
+ // True when either the Target Machine specifies no optimizations or the
+ // function has the optnone attribute.
+ bool EnableOpts = false;
+
+ /// Switch analysis and optimization.
+ class GISelSwitchLowering : public SwitchCG::SwitchLowering {
+ public:
+ GISelSwitchLowering(IRTranslator *irt, FunctionLoweringInfo &funcinfo)
+ : SwitchLowering(funcinfo), IRT(irt) {
+ assert(irt && "irt is null!");
+ }
+
+ virtual void addSuccessorWithProb(
+ MachineBasicBlock *Src, MachineBasicBlock *Dst,
+ BranchProbability Prob = BranchProbability::getUnknown()) override {
+ IRT->addSuccessorWithProb(Src, Dst, Prob);
+ }
+
+ virtual ~GISelSwitchLowering() = default;
+
+ private:
+ IRTranslator *IRT;
+ };
+
+ std::unique_ptr<GISelSwitchLowering> SL;
+
// * Insert all the code needed to materialize the constants
// at the proper place. E.g., Entry block or dominator block
// of each constant depending on how fancy we want to be.
// * Clear the different maps.
void finalizeFunction();
+ // Handle emitting jump tables for each basic block.
+ void finalizeBasicBlock();
+
/// Get the VRegs that represent \p Val.
/// Non-aggregate types have just one corresponding VReg and the list can be
/// used as a single "unsigned". Aggregates get flattened. If such VRegs do
/// not exist, they are created.
- ArrayRef<unsigned> getOrCreateVRegs(const Value &Val);
+ ArrayRef<Register> getOrCreateVRegs(const Value &Val);
- unsigned getOrCreateVReg(const Value &Val) {
+ Register getOrCreateVReg(const Value &Val) {
auto Regs = getOrCreateVRegs(Val);
if (Regs.empty())
return 0;
@@ -534,6 +595,14 @@
return SmallVector<MachineBasicBlock *, 4>(1, &getMBB(*Edge.first));
}
+ /// Return branch probability calculated by BranchProbabilityInfo for IR
+ /// blocks.
+ BranchProbability getEdgeProbability(const MachineBasicBlock *Src,
+ const MachineBasicBlock *Dst) const;
+
+ void addSuccessorWithProb(MachineBasicBlock *Src, MachineBasicBlock *Dst,
+ BranchProbability Prob);
+
public:
// Ctor, nothing fancy.
IRTranslator();
diff --git a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
index 87ef2f8..e010180 100644
--- a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
+++ b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
@@ -437,15 +437,15 @@
unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
- MMO->getSize() * 8 != Size) {
+ MMO->getSizeInBits() != Size) {
if (handleReject() == RejectAndGiveUp)
return false;
} else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
- MMO->getSize() * 8 >= Size) {
+ MMO->getSizeInBits() >= Size) {
if (handleReject() == RejectAndGiveUp)
return false;
} else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
- MMO->getSize() * 8 <= Size)
+ MMO->getSizeInBits() <= Size)
if (handleReject() == RejectAndGiveUp)
return false;
@@ -478,17 +478,19 @@
<< InsnID << "]->getOperand(" << OpIdx
<< "), SizeInBits=" << SizeInBits << ")\n");
assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
+ MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
+ const LLT Ty = MRI.getType(MO.getReg());
+
// iPTR must be looked up in the target.
if (SizeInBits == 0) {
MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
- SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
+ const unsigned AddrSpace = Ty.getAddressSpace();
+ SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
}
assert(SizeInBits != 0 && "Pointer size must be known");
- MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
if (MO.isReg()) {
- const LLT &Ty = MRI.getType(MO.getReg());
if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
if (handleReject() == RejectAndGiveUp)
return false;
diff --git a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
index e7680e1..a22778b 100644
--- a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
+++ b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
@@ -28,6 +28,18 @@
MachineRegisterInfo &MRI;
const LegalizerInfo &LI;
+ static bool isArtifactCast(unsigned Opc) {
+ switch (Opc) {
+ case TargetOpcode::G_TRUNC:
+ case TargetOpcode::G_SEXT:
+ case TargetOpcode::G_ZEXT:
+ case TargetOpcode::G_ANYEXT:
+ return true;
+ default:
+ return false;
+ }
+ }
+
public:
LegalizationArtifactCombiner(MachineIRBuilder &B, MachineRegisterInfo &MRI,
const LegalizerInfo &LI)
@@ -39,11 +51,11 @@
return false;
Builder.setInstr(MI);
- unsigned DstReg = MI.getOperand(0).getReg();
- unsigned SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg());
+ Register DstReg = MI.getOperand(0).getReg();
+ Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg());
// aext(trunc x) - > aext/copy/trunc x
- unsigned TruncSrc;
+ Register TruncSrc;
if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) {
LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;);
Builder.buildAnyExtOrTrunc(DstReg, TruncSrc);
@@ -52,7 +64,7 @@
}
// aext([asz]ext x) -> [asz]ext x
- unsigned ExtSrc;
+ Register ExtSrc;
MachineInstr *ExtMI;
if (mi_match(SrcReg, MRI,
m_all_of(m_MInstr(ExtMI), m_any_of(m_GAnyExt(m_Reg(ExtSrc)),
@@ -62,6 +74,20 @@
markInstAndDefDead(MI, *ExtMI, DeadInsts);
return true;
}
+
+ // Try to fold aext(g_constant) when the larger constant type is legal.
+ // Can't use MIPattern because we don't have a specific constant in mind.
+ auto *SrcMI = MRI.getVRegDef(SrcReg);
+ if (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT) {
+ const LLT &DstTy = MRI.getType(DstReg);
+ if (isInstLegal({TargetOpcode::G_CONSTANT, {DstTy}})) {
+ auto &CstVal = SrcMI->getOperand(1);
+ Builder.buildConstant(
+ DstReg, CstVal.getCImm()->getValue().sext(DstTy.getSizeInBits()));
+ markInstAndDefDead(MI, *SrcMI, DeadInsts);
+ return true;
+ }
+ }
return tryFoldImplicitDef(MI, DeadInsts);
}
@@ -72,11 +98,11 @@
return false;
Builder.setInstr(MI);
- unsigned DstReg = MI.getOperand(0).getReg();
- unsigned SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg());
+ Register DstReg = MI.getOperand(0).getReg();
+ Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg());
// zext(trunc x) - > and (aext/copy/trunc x), mask
- unsigned TruncSrc;
+ Register TruncSrc;
if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) {
LLT DstTy = MRI.getType(DstReg);
if (isInstUnsupported({TargetOpcode::G_AND, {DstTy}}) ||
@@ -91,6 +117,20 @@
markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts);
return true;
}
+
+ // Try to fold zext(g_constant) when the larger constant type is legal.
+ // Can't use MIPattern because we don't have a specific constant in mind.
+ auto *SrcMI = MRI.getVRegDef(SrcReg);
+ if (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT) {
+ const LLT &DstTy = MRI.getType(DstReg);
+ if (isInstLegal({TargetOpcode::G_CONSTANT, {DstTy}})) {
+ auto &CstVal = SrcMI->getOperand(1);
+ Builder.buildConstant(
+ DstReg, CstVal.getCImm()->getValue().zext(DstTy.getSizeInBits()));
+ markInstAndDefDead(MI, *SrcMI, DeadInsts);
+ return true;
+ }
+ }
return tryFoldImplicitDef(MI, DeadInsts);
}
@@ -101,11 +141,11 @@
return false;
Builder.setInstr(MI);
- unsigned DstReg = MI.getOperand(0).getReg();
- unsigned SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg());
+ Register DstReg = MI.getOperand(0).getReg();
+ Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg());
// sext(trunc x) - > ashr (shl (aext/copy/trunc x), c), c
- unsigned TruncSrc;
+ Register TruncSrc;
if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) {
LLT DstTy = MRI.getType(DstReg);
// Guess on the RHS shift amount type, which should be re-legalized if
@@ -139,7 +179,7 @@
if (MachineInstr *DefMI = getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF,
MI.getOperand(1).getReg(), MRI)) {
Builder.setInstr(MI);
- unsigned DstReg = MI.getOperand(0).getReg();
+ Register DstReg = MI.getOperand(0).getReg();
LLT DstTy = MRI.getType(DstReg);
if (Opcode == TargetOpcode::G_ANYEXT) {
@@ -180,21 +220,33 @@
return false;
unsigned NumDefs = MI.getNumOperands() - 1;
+ MachineInstr *SrcDef =
+ getDefIgnoringCopies(MI.getOperand(NumDefs).getReg(), MRI);
+ if (!SrcDef)
+ return false;
LLT OpTy = MRI.getType(MI.getOperand(NumDefs).getReg());
LLT DestTy = MRI.getType(MI.getOperand(0).getReg());
+ MachineInstr *MergeI = SrcDef;
+ unsigned ConvertOp = 0;
+ // Handle intermediate conversions
+ unsigned SrcOp = SrcDef->getOpcode();
+ if (isArtifactCast(SrcOp)) {
+ ConvertOp = SrcOp;
+ MergeI = getDefIgnoringCopies(SrcDef->getOperand(1).getReg(), MRI);
+ }
+
+ // FIXME: Handle scalarizing concat_vectors (scalar result type with vector
+ // source)
unsigned MergingOpcode = getMergeOpcode(OpTy, DestTy);
- MachineInstr *MergeI =
- getOpcodeDef(MergingOpcode, MI.getOperand(NumDefs).getReg(), MRI);
-
- if (!MergeI)
+ if (!MergeI || MergeI->getOpcode() != MergingOpcode)
return false;
const unsigned NumMergeRegs = MergeI->getNumOperands() - 1;
if (NumMergeRegs < NumDefs) {
- if (NumDefs % NumMergeRegs != 0)
+ if (ConvertOp != 0 || NumDefs % NumMergeRegs != 0)
return false;
Builder.setInstr(MI);
@@ -207,7 +259,7 @@
const unsigned NewNumDefs = NumDefs / NumMergeRegs;
for (unsigned Idx = 0; Idx < NumMergeRegs; ++Idx) {
- SmallVector<unsigned, 2> DstRegs;
+ SmallVector<Register, 2> DstRegs;
for (unsigned j = 0, DefIdx = Idx * NewNumDefs; j < NewNumDefs;
++j, ++DefIdx)
DstRegs.push_back(MI.getOperand(DefIdx).getReg());
@@ -216,7 +268,7 @@
}
} else if (NumMergeRegs > NumDefs) {
- if (NumMergeRegs % NumDefs != 0)
+ if (ConvertOp != 0 || NumMergeRegs % NumDefs != 0)
return false;
Builder.setInstr(MI);
@@ -229,7 +281,7 @@
const unsigned NumRegs = NumMergeRegs / NumDefs;
for (unsigned DefIdx = 0; DefIdx < NumDefs; ++DefIdx) {
- SmallVector<unsigned, 2> Regs;
+ SmallVector<Register, 2> Regs;
for (unsigned j = 0, Idx = NumRegs * DefIdx + 1; j < NumRegs;
++j, ++Idx)
Regs.push_back(MergeI->getOperand(Idx).getReg());
@@ -238,10 +290,22 @@
}
} else {
+ LLT MergeSrcTy = MRI.getType(MergeI->getOperand(1).getReg());
+ if (ConvertOp) {
+ Builder.setInstr(MI);
+
+ for (unsigned Idx = 0; Idx < NumDefs; ++Idx) {
+ Register MergeSrc = MergeI->getOperand(Idx + 1).getReg();
+ Builder.buildInstr(ConvertOp, {MI.getOperand(Idx).getReg()},
+ {MergeSrc});
+ }
+
+ markInstAndDefDead(MI, *MergeI, DeadInsts);
+ return true;
+ }
// FIXME: is a COPY appropriate if the types mismatch? We know both
// registers are allocatable by now.
- if (MRI.getType(MI.getOperand(0).getReg()) !=
- MRI.getType(MergeI->getOperand(1).getReg()))
+ if (DestTy != MergeSrcTy)
return false;
for (unsigned Idx = 0; Idx < NumDefs; ++Idx)
@@ -317,7 +381,13 @@
/// Adds instructions that are dead as a result of the combine
/// into DeadInsts, which can include MI.
bool tryCombineInstruction(MachineInstr &MI,
- SmallVectorImpl<MachineInstr *> &DeadInsts) {
+ SmallVectorImpl<MachineInstr *> &DeadInsts,
+ GISelObserverWrapper &WrapperObserver) {
+ // This might be a recursive call, and we might have DeadInsts already
+ // populated. To avoid bad things happening later with multiple vreg defs
+ // etc, process the dead instructions now if any.
+ if (!DeadInsts.empty())
+ deleteMarkedDeadInsts(DeadInsts, WrapperObserver);
switch (MI.getOpcode()) {
default:
return false;
@@ -334,7 +404,7 @@
case TargetOpcode::G_TRUNC: {
bool Changed = false;
for (auto &Use : MRI.use_instructions(MI.getOperand(0).getReg()))
- Changed |= tryCombineInstruction(Use, DeadInsts);
+ Changed |= tryCombineInstruction(Use, DeadInsts, WrapperObserver);
return Changed;
}
}
@@ -383,8 +453,10 @@
MachineInstr *TmpDef = MRI.getVRegDef(PrevRegSrc);
if (MRI.hasOneUse(PrevRegSrc)) {
if (TmpDef != &DefMI) {
- assert(TmpDef->getOpcode() == TargetOpcode::COPY &&
- "Expecting copy here");
+ assert((TmpDef->getOpcode() == TargetOpcode::COPY ||
+ isArtifactCast(TmpDef->getOpcode())) &&
+ "Expecting copy or artifact cast here");
+
DeadInsts.push_back(TmpDef);
}
} else
@@ -395,6 +467,22 @@
DeadInsts.push_back(&DefMI);
}
+ /// Erase the dead instructions in the list and call the observer hooks.
+ /// Normally the Legalizer will deal with erasing instructions that have been
+ /// marked dead. However, for the trunc(ext(x)) cases we can end up trying to
+ /// process instructions which have been marked dead, but otherwise break the
+ /// MIR by introducing multiple vreg defs. For those cases, allow the combines
+ /// to explicitly delete the instructions before we run into trouble.
+ void deleteMarkedDeadInsts(SmallVectorImpl<MachineInstr *> &DeadInsts,
+ GISelObserverWrapper &WrapperObserver) {
+ for (auto *DeadMI : DeadInsts) {
+ LLVM_DEBUG(dbgs() << *DeadMI << "Is dead, eagerly deleting\n");
+ WrapperObserver.erasingInstr(*DeadMI);
+ DeadMI->eraseFromParentAndMarkDBGValuesForRemoval();
+ }
+ DeadInsts.clear();
+ }
+
/// Checks if the target legalizer info has specified anything about the
/// instruction, or if unsupported.
bool isInstUnsupported(const LegalityQuery &Query) const {
@@ -403,6 +491,10 @@
return Step.Action == Unsupported || Step.Action == NotFound;
}
+ bool isInstLegal(const LegalityQuery &Query) const {
+ return LI.getAction(Query).Action == LegalizeActions::Legal;
+ }
+
bool isConstantUnsupported(LLT Ty) const {
if (!Ty.isVector())
return isInstUnsupported({TargetOpcode::G_CONSTANT, {Ty}});
@@ -414,8 +506,8 @@
/// Looks through copy instructions and returns the actual
/// source register.
- unsigned lookThroughCopyInstrs(unsigned Reg) {
- unsigned TmpReg;
+ unsigned lookThroughCopyInstrs(Register Reg) {
+ Register TmpReg;
while (mi_match(Reg, MRI, m_Copy(m_Reg(TmpReg)))) {
if (MRI.getType(TmpReg).isValid())
Reg = TmpReg;
diff --git a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/Legalizer.h b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/Legalizer.h
index 0181650..13cf3f7 100644
--- a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/Legalizer.h
+++ b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/Legalizer.h
@@ -54,6 +54,11 @@
MachineFunctionProperties::Property::Legalized);
}
+ MachineFunctionProperties getClearedProperties() const override {
+ return MachineFunctionProperties()
+ .set(MachineFunctionProperties::Property::NoPHIs);
+ }
+
bool combineExtracts(MachineInstr &MI, MachineRegisterInfo &MRI,
const TargetInstrInfo &TII);
diff --git a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index 136356c..e8cb65f 100644
--- a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -124,6 +124,11 @@
/// result elements, and replacing the vreg of the operand in place.
void moreElementsVectorDst(MachineInstr &MI, LLT MoreTy, unsigned OpIdx);
+ /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a
+ /// Use by producing a vector with undefined high elements, extracting the
+ /// original vector type, and replacing the vreg of the operand in place.
+ void moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy, unsigned OpIdx);
+
LegalizeResult
widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy);
LegalizeResult
@@ -136,14 +141,14 @@
/// Helper function to split a wide generic register into bitwise blocks with
/// the given Type (which implies the number of blocks needed). The generic
/// registers created are appended to Ops, starting at bit 0 of Reg.
- void extractParts(unsigned Reg, LLT Ty, int NumParts,
- SmallVectorImpl<unsigned> &VRegs);
+ void extractParts(Register Reg, LLT Ty, int NumParts,
+ SmallVectorImpl<Register> &VRegs);
/// Version which handles irregular splits.
- bool extractParts(unsigned Reg, LLT RegTy, LLT MainTy,
+ bool extractParts(Register Reg, LLT RegTy, LLT MainTy,
LLT &LeftoverTy,
- SmallVectorImpl<unsigned> &VRegs,
- SmallVectorImpl<unsigned> &LeftoverVRegs);
+ SmallVectorImpl<Register> &VRegs,
+ SmallVectorImpl<Register> &LeftoverVRegs);
/// Helper function to build a wide generic register \p DstReg of type \p
/// RegTy from smaller parts. This will produce a G_MERGE_VALUES,
@@ -154,9 +159,16 @@
///
/// If \p ResultTy does not evenly break into \p PartTy sized pieces, the
/// remainder must be specified with \p LeftoverRegs of type \p LeftoverTy.
- void insertParts(unsigned DstReg, LLT ResultTy,
- LLT PartTy, ArrayRef<unsigned> PartRegs,
- LLT LeftoverTy = LLT(), ArrayRef<unsigned> LeftoverRegs = {});
+ void insertParts(Register DstReg, LLT ResultTy,
+ LLT PartTy, ArrayRef<Register> PartRegs,
+ LLT LeftoverTy = LLT(), ArrayRef<Register> LeftoverRegs = {});
+
+ /// Perform generic multiplication of values held in multiple registers.
+ /// Generated instructions use only types NarrowTy and i1.
+ /// Destination can be same or two times size of the source.
+ void multiplyRegisters(SmallVectorImpl<Register> &DstRegs,
+ ArrayRef<Register> Src1Regs,
+ ArrayRef<Register> Src2Regs, LLT NarrowTy);
LegalizeResult fewerElementsVectorImplicitDef(MachineInstr &MI,
unsigned TypeIdx, LLT NarrowTy);
@@ -181,6 +193,12 @@
LegalizeResult
fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy);
+ LegalizeResult fewerElementsVectorPhi(MachineInstr &MI,
+ unsigned TypeIdx, LLT NarrowTy);
+
+ LegalizeResult moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
+ LLT MoreTy);
+
LegalizeResult
reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy);
@@ -188,14 +206,21 @@
LLT HalfTy, LLT ShiftAmtTy);
LegalizeResult narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
- LegalizeResult narrowScalarMul(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
+ LegalizeResult narrowScalarMul(MachineInstr &MI, LLT Ty);
LegalizeResult narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
LegalizeResult narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
+ LegalizeResult narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
LegalizeResult narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
LegalizeResult lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
+ LegalizeResult lowerU64ToF32BitOps(MachineInstr &MI);
+ LegalizeResult lowerUITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
+ LegalizeResult lowerSITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
+ LegalizeResult lowerMinMax(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
+ LegalizeResult lowerFCopySign(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
+
MachineRegisterInfo &MRI;
const LegalizerInfo &LI;
/// To keep track of changes made by the LegalizerHelper.
diff --git a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
index 5979173..513c98f 100644
--- a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
+++ b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
@@ -92,6 +92,7 @@
UseLegacyRules,
};
} // end namespace LegalizeActions
+raw_ostream &operator<<(raw_ostream &OS, LegalizeActions::LegalizeAction Action);
using LegalizeActions::LegalizeAction;
@@ -122,6 +123,7 @@
struct MemDesc {
uint64_t SizeInBits;
+ uint64_t AlignInBits;
AtomicOrdering Ordering;
};
@@ -164,13 +166,23 @@
std::function<std::pair<unsigned, LLT>(const LegalityQuery &)>;
namespace LegalityPredicates {
-struct TypePairAndMemSize {
+struct TypePairAndMemDesc {
LLT Type0;
LLT Type1;
uint64_t MemSize;
+ uint64_t Align;
- bool operator==(const TypePairAndMemSize &Other) const {
+ bool operator==(const TypePairAndMemDesc &Other) const {
return Type0 == Other.Type0 && Type1 == Other.Type1 &&
+ Align == Other.Align &&
+ MemSize == Other.MemSize;
+ }
+
+ /// \returns true if this memory access is legal with for the acecss described
+ /// by \p Other (The alignment is sufficient for the size and result type).
+ bool isCompatible(const TypePairAndMemDesc &Other) const {
+ return Type0 == Other.Type0 && Type1 == Other.Type1 &&
+ Align >= Other.Align &&
MemSize == Other.MemSize;
}
};
@@ -199,9 +211,9 @@
std::initializer_list<std::pair<LLT, LLT>> TypesInit);
/// True iff the given types for the given pair of type indexes is one of the
/// specified type pairs.
-LegalityPredicate typePairAndMemSizeInSet(
+LegalityPredicate typePairAndMemDescInSet(
unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx,
- std::initializer_list<TypePairAndMemSize> TypesAndMemSizeInit);
+ std::initializer_list<TypePairAndMemDesc> TypesAndMemDescInit);
/// True iff the specified type index is a scalar.
LegalityPredicate isScalar(unsigned TypeIdx);
/// True iff the specified type index is a vector.
@@ -455,13 +467,13 @@
return actionFor(LegalizeAction::Legal, Types);
}
/// The instruction is legal when type indexes 0 and 1 along with the memory
- /// size is any type and size tuple in the given list.
- LegalizeRuleSet &legalForTypesWithMemSize(
- std::initializer_list<LegalityPredicates::TypePairAndMemSize>
- TypesAndMemSize) {
+ /// size and minimum alignment is any type and size tuple in the given list.
+ LegalizeRuleSet &legalForTypesWithMemDesc(
+ std::initializer_list<LegalityPredicates::TypePairAndMemDesc>
+ TypesAndMemDesc) {
return actionIf(LegalizeAction::Legal,
- LegalityPredicates::typePairAndMemSizeInSet(
- typeIdx(0), typeIdx(1), /*MMOIdx*/ 0, TypesAndMemSize));
+ LegalityPredicates::typePairAndMemDescInSet(
+ typeIdx(0), typeIdx(1), /*MMOIdx*/ 0, TypesAndMemDesc));
}
/// The instruction is legal when type indexes 0 and 1 are both in the given
/// list. That is, the type pair is in the cartesian product of the list.
@@ -474,6 +486,20 @@
std::initializer_list<LLT> Types1) {
return actionForCartesianProduct(LegalizeAction::Legal, Types0, Types1);
}
+ /// The instruction is legal when type indexes 0, 1, and 2 are both their
+ /// respective lists.
+ LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0,
+ std::initializer_list<LLT> Types1,
+ std::initializer_list<LLT> Types2) {
+ return actionForCartesianProduct(LegalizeAction::Legal, Types0, Types1,
+ Types2);
+ }
+
+ LegalizeRuleSet &alwaysLegal() {
+ using namespace LegalizeMutations;
+ markAllTypeIdxsAsCovered();
+ return actionIf(LegalizeAction::Legal, always);
+ }
/// The instruction is lowered.
LegalizeRuleSet &lower() {
@@ -624,6 +650,13 @@
LegalizeRuleSet &customFor(std::initializer_list<LLT> Types) {
return actionFor(LegalizeAction::Custom, Types);
}
+
+ /// The instruction is custom when type indexes 0 and 1 is any type pair in the
+ /// given list.
+ LegalizeRuleSet &customFor(std::initializer_list<std::pair<LLT, LLT>> Types) {
+ return actionFor(LegalizeAction::Custom, Types);
+ }
+
LegalizeRuleSet &customForCartesianProduct(std::initializer_list<LLT> Types) {
return actionForCartesianProduct(LegalizeAction::Custom, Types);
}
@@ -670,7 +703,7 @@
LegalizeMutations::scalarize(TypeIdx));
}
- /// Ensure the scalar is at least as wide as Ty.
+ /// Ensure the scalar or element is at least as wide as Ty.
LegalizeRuleSet &minScalarOrElt(unsigned TypeIdx, const LLT &Ty) {
using namespace LegalityPredicates;
using namespace LegalizeMutations;
@@ -679,6 +712,17 @@
changeElementTo(typeIdx(TypeIdx), Ty));
}
+ /// Ensure the scalar or element is at least as wide as Ty.
+ LegalizeRuleSet &minScalarOrEltIf(LegalityPredicate Predicate,
+ unsigned TypeIdx, const LLT &Ty) {
+ using namespace LegalityPredicates;
+ using namespace LegalizeMutations;
+ return actionIf(LegalizeAction::WidenScalar,
+ all(Predicate, scalarOrEltNarrowerThan(
+ TypeIdx, Ty.getScalarSizeInBits())),
+ changeElementTo(typeIdx(TypeIdx), Ty));
+ }
+
/// Ensure the scalar is at least as wide as Ty.
LegalizeRuleSet &minScalar(unsigned TypeIdx, const LLT &Ty) {
using namespace LegalityPredicates;
@@ -749,6 +793,22 @@
});
}
+ /// Conditionally widen the scalar or elt to match the size of another.
+ LegalizeRuleSet &minScalarEltSameAsIf(LegalityPredicate Predicate,
+ unsigned TypeIdx, unsigned LargeTypeIdx) {
+ typeIdx(TypeIdx);
+ return widenScalarIf(
+ [=](const LegalityQuery &Query) {
+ return Query.Types[LargeTypeIdx].getScalarSizeInBits() >
+ Query.Types[TypeIdx].getScalarSizeInBits() &&
+ Predicate(Query);
+ },
+ [=](const LegalityQuery &Query) {
+ LLT T = Query.Types[LargeTypeIdx];
+ return std::make_pair(TypeIdx, T);
+ });
+ }
+
/// Add more elements to the vector to reach the next power of two.
/// No effect if the type is not a vector or the element count is a power of
/// two.
@@ -1043,12 +1103,22 @@
LegalizeActionStep getAction(const MachineInstr &MI,
const MachineRegisterInfo &MRI) const;
+ bool isLegal(const LegalityQuery &Query) const {
+ return getAction(Query).Action == LegalizeAction::Legal;
+ }
bool isLegal(const MachineInstr &MI, const MachineRegisterInfo &MRI) const;
+ bool isLegalOrCustom(const MachineInstr &MI,
+ const MachineRegisterInfo &MRI) const;
virtual bool legalizeCustom(MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &MIRBuilder,
GISelChangeObserver &Observer) const;
+ /// Return true if MI is either legal or has been legalized and false
+ /// if not legal.
+ virtual bool legalizeIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI,
+ MachineIRBuilder &MIRBuilder) const;
+
private:
/// Determine what action should be taken to legalize the given generic
/// instruction opcode, type-index and type. Requires computeTables to have
diff --git a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/Localizer.h b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/Localizer.h
index cfc7c35..06de580 100644
--- a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/Localizer.h
+++ b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/Localizer.h
@@ -21,12 +21,14 @@
#ifndef LLVM_CODEGEN_GLOBALISEL_LOCALIZER_H
#define LLVM_CODEGEN_GLOBALISEL_LOCALIZER_H
+#include "llvm/ADT/SetVector.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
namespace llvm {
// Forward declarations.
class MachineRegisterInfo;
+class TargetTransformInfo;
/// This pass implements the localization mechanism described at the
/// top of this file. One specificity of the implementation is that
@@ -43,9 +45,11 @@
/// MRI contains all the register class/bank information that this
/// pass uses and updates.
MachineRegisterInfo *MRI;
+ /// TTI used for getting remat costs for instructions.
+ TargetTransformInfo *TTI;
/// Check whether or not \p MI needs to be moved close to its uses.
- static bool shouldLocalize(const MachineInstr &MI);
+ bool shouldLocalize(const MachineInstr &MI);
/// Check if \p MOUse is used in the same basic block as \p Def.
/// If the use is in the same block, we say it is local.
@@ -57,6 +61,15 @@
/// Initialize the field members using \p MF.
void init(MachineFunction &MF);
+ typedef SmallSetVector<MachineInstr *, 32> LocalizedSetVecT;
+
+ /// Do inter-block localization from the entry block.
+ bool localizeInterBlock(MachineFunction &MF,
+ LocalizedSetVecT &LocalizedInstrs);
+
+ /// Do intra-block localization of already localized instructions.
+ bool localizeIntraBlock(LocalizedSetVecT &LocalizedInstrs);
+
public:
Localizer();
diff --git a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h
index 1f4ee8a..13eddd9 100644
--- a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h
+++ b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h
@@ -1,4 +1,4 @@
-//== ----- llvm/CodeGen/GlobalISel/MIPatternMatch.h --------------------- == //
+//==------ llvm/CodeGen/GlobalISel/MIPatternMatch.h -------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -30,8 +30,7 @@
SubPatternT SubPat;
OneUse_match(const SubPatternT &SP) : SubPat(SP) {}
- template <typename OpTy>
- bool match(const MachineRegisterInfo &MRI, unsigned Reg) {
+ bool match(MachineRegisterInfo &MRI, unsigned Reg) {
return MRI.hasOneUse(Reg) && SubPat.match(MRI, Reg);
}
};
@@ -161,7 +160,7 @@
}
};
-inline bind_ty<unsigned> m_Reg(unsigned &R) { return R; }
+inline bind_ty<Register> m_Reg(Register &R) { return R; }
inline bind_ty<MachineInstr *> m_MInstr(MachineInstr *&MI) { return MI; }
inline bind_ty<LLT> m_Type(LLT &Ty) { return Ty; }
diff --git a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index 6e1ff7c..da3c478 100644
--- a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -59,13 +59,14 @@
class DstOp {
union {
LLT LLTTy;
- unsigned Reg;
+ Register Reg;
const TargetRegisterClass *RC;
};
public:
enum class DstType { Ty_LLT, Ty_Reg, Ty_RC };
DstOp(unsigned R) : Reg(R), Ty(DstType::Ty_Reg) {}
+ DstOp(Register R) : Reg(R), Ty(DstType::Ty_Reg) {}
DstOp(const MachineOperand &Op) : Reg(Op.getReg()), Ty(DstType::Ty_Reg) {}
DstOp(const LLT &T) : LLTTy(T), Ty(DstType::Ty_LLT) {}
DstOp(const TargetRegisterClass *TRC) : RC(TRC), Ty(DstType::Ty_RC) {}
@@ -96,7 +97,7 @@
llvm_unreachable("Unrecognised DstOp::DstType enum");
}
- unsigned getReg() const {
+ Register getReg() const {
assert(Ty == DstType::Ty_Reg && "Not a register");
return Reg;
}
@@ -119,13 +120,13 @@
class SrcOp {
union {
MachineInstrBuilder SrcMIB;
- unsigned Reg;
+ Register Reg;
CmpInst::Predicate Pred;
};
public:
enum class SrcType { Ty_Reg, Ty_MIB, Ty_Predicate };
- SrcOp(unsigned R) : Reg(R), Ty(SrcType::Ty_Reg) {}
+ SrcOp(Register R) : Reg(R), Ty(SrcType::Ty_Reg) {}
SrcOp(const MachineOperand &Op) : Reg(Op.getReg()), Ty(SrcType::Ty_Reg) {}
SrcOp(const MachineInstrBuilder &MIB) : SrcMIB(MIB), Ty(SrcType::Ty_MIB) {}
SrcOp(const CmpInst::Predicate P) : Pred(P), Ty(SrcType::Ty_Predicate) {}
@@ -156,7 +157,7 @@
llvm_unreachable("Unrecognised SrcOp::SrcType enum");
}
- unsigned getReg() const {
+ Register getReg() const {
switch (Ty) {
case SrcType::Ty_Predicate:
llvm_unreachable("Not a register operand");
@@ -237,6 +238,10 @@
return *State.MF;
}
+ const DataLayout &getDataLayout() const {
+ return getMF().getFunction().getParent()->getDataLayout();
+ }
+
/// Getter for DebugLoc
const DebugLoc &getDL() { return State.DL; }
@@ -317,13 +322,13 @@
/// Build and insert a DBG_VALUE instruction expressing the fact that the
/// associated \p Variable lives in \p Reg (suitably modified by \p Expr).
- MachineInstrBuilder buildDirectDbgValue(unsigned Reg, const MDNode *Variable,
+ MachineInstrBuilder buildDirectDbgValue(Register Reg, const MDNode *Variable,
const MDNode *Expr);
/// Build and insert a DBG_VALUE instruction expressing the fact that the
/// associated \p Variable lives in memory at \p Reg (suitably modified by \p
/// Expr).
- MachineInstrBuilder buildIndirectDbgValue(unsigned Reg,
+ MachineInstrBuilder buildIndirectDbgValue(Register Reg,
const MDNode *Variable,
const MDNode *Expr);
@@ -352,7 +357,7 @@
/// \pre \p Res must be a generic virtual register with pointer type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildFrameIndex(unsigned Res, int Idx);
+ MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx);
/// Build and insert \p Res = G_GLOBAL_VALUE \p GV
///
@@ -364,8 +369,7 @@
/// in the same address space as \p GV.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildGlobalValue(unsigned Res, const GlobalValue *GV);
-
+ MachineInstrBuilder buildGlobalValue(const DstOp &Res, const GlobalValue *GV);
/// Build and insert \p Res = G_GEP \p Op0, \p Op1
///
@@ -378,8 +382,8 @@
/// \pre \p Op1 must be a generic virtual register with scalar type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildGEP(unsigned Res, unsigned Op0,
- unsigned Op1);
+ MachineInstrBuilder buildGEP(const DstOp &Res, const SrcOp &Op0,
+ const SrcOp &Op1);
/// Materialize and insert \p Res = G_GEP \p Op0, (G_CONSTANT \p Value)
///
@@ -397,7 +401,7 @@
/// type as \p Op0 or \p Op0 itself.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- Optional<MachineInstrBuilder> materializeGEP(unsigned &Res, unsigned Op0,
+ Optional<MachineInstrBuilder> materializeGEP(Register &Res, Register Op0,
const LLT &ValueTy,
uint64_t Value);
@@ -414,9 +418,24 @@
/// be cleared in \p Op0.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildPtrMask(unsigned Res, unsigned Op0,
+ MachineInstrBuilder buildPtrMask(const DstOp &Res, const SrcOp &Op0,
uint32_t NumBits);
+ /// Build and insert \p Res, \p CarryOut = G_UADDO \p Op0, \p Op1
+ ///
+ /// G_UADDO sets \p Res to \p Op0 + \p Op1 (truncated to the bit width) and
+ /// sets \p CarryOut to 1 if the result overflowed in unsigned arithmetic.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p Res, \p Op0 and \p Op1 must be generic virtual registers with the
+ /// same scalar type.
+ ////\pre \p CarryOut must be generic virtual register with scalar type
+ ///(typically s1)
+ ///
+ /// \return The newly created instruction.
+ MachineInstrBuilder buildUAddo(const DstOp &Res, const DstOp &CarryOut,
+ const SrcOp &Op0, const SrcOp &Op1);
+
/// Build and insert \p Res, \p CarryOut = G_UADDE \p Op0,
/// \p Op1, \p CarryIn
///
@@ -465,6 +484,16 @@
/// \return The newly created instruction.
MachineInstrBuilder buildSExt(const DstOp &Res, const SrcOp &Op);
+ /// Build and insert a G_PTRTOINT instruction.
+ MachineInstrBuilder buildPtrToInt(const DstOp &Dst, const SrcOp &Src) {
+ return buildInstr(TargetOpcode::G_PTRTOINT, {Dst}, {Src});
+ }
+
+ /// Build and insert \p Dst = G_BITCAST \p Src
+ MachineInstrBuilder buildBitcast(const DstOp &Dst, const SrcOp &Src) {
+ return buildInstr(TargetOpcode::G_BITCAST, {Dst}, {Src});
+ }
+
/// \return The opcode of the extension the target wants to use for boolean
/// values.
unsigned getBoolExtOp(bool IsVec, bool IsFP) const;
@@ -554,7 +583,7 @@
/// depend on bit 0 (for now).
///
/// \return The newly created instruction.
- MachineInstrBuilder buildBrCond(unsigned Tst, MachineBasicBlock &Dest);
+ MachineInstrBuilder buildBrCond(Register Tst, MachineBasicBlock &Dest);
/// Build and insert G_BRINDIRECT \p Tgt
///
@@ -564,7 +593,21 @@
/// \pre \p Tgt must be a generic virtual register with pointer type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildBrIndirect(unsigned Tgt);
+ MachineInstrBuilder buildBrIndirect(Register Tgt);
+
+ /// Build and insert G_BRJT \p TablePtr, \p JTI, \p IndexReg
+ ///
+ /// G_BRJT is a jump table branch using a table base pointer \p TablePtr,
+ /// jump table index \p JTI and index \p IndexReg
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p TablePtr must be a generic virtual register with pointer type.
+ /// \pre \p JTI must be be a jump table index.
+ /// \pre \p IndexReg must be a generic virtual register with pointer type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildBrJT(Register TablePtr, unsigned JTI,
+ Register IndexReg);
/// Build and insert \p Res = G_CONSTANT \p Val
///
@@ -603,6 +646,7 @@
const ConstantFP &Val);
MachineInstrBuilder buildFConstant(const DstOp &Res, double Val);
+ MachineInstrBuilder buildFConstant(const DstOp &Res, const APFloat &Val);
/// Build and insert \p Res = COPY Op
///
@@ -622,7 +666,7 @@
/// \pre \p Addr must be a generic virtual register with pointer type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildLoad(unsigned Res, unsigned Addr,
+ MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr,
MachineMemOperand &MMO);
/// Build and insert `Res = <opcode> Addr, MMO`.
@@ -634,8 +678,8 @@
/// \pre \p Addr must be a generic virtual register with pointer type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildLoadInstr(unsigned Opcode, unsigned Res,
- unsigned Addr, MachineMemOperand &MMO);
+ MachineInstrBuilder buildLoadInstr(unsigned Opcode, const DstOp &Res,
+ const SrcOp &Addr, MachineMemOperand &MMO);
/// Build and insert `G_STORE Val, Addr, MMO`.
///
@@ -646,7 +690,7 @@
/// \pre \p Addr must be a generic virtual register with pointer type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildStore(unsigned Val, unsigned Addr,
+ MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr,
MachineMemOperand &MMO);
/// Build and insert `Res0, ... = G_EXTRACT Src, Idx0`.
@@ -655,7 +699,7 @@
/// \pre \p Res and \p Src must be generic virtual registers.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildExtract(unsigned Res, unsigned Src, uint64_t Index);
+ MachineInstrBuilder buildExtract(const DstOp &Res, const SrcOp &Src, uint64_t Index);
/// Build and insert \p Res = IMPLICIT_DEF.
MachineInstrBuilder buildUndef(const DstOp &Res);
@@ -673,7 +717,7 @@
/// \pre The bits defined by each Op (derived from index and scalar size) must
/// not overlap.
/// \pre \p Indices must be in ascending order of bit position.
- void buildSequence(unsigned Res, ArrayRef<unsigned> Ops,
+ void buildSequence(Register Res, ArrayRef<Register> Ops,
ArrayRef<uint64_t> Indices);
/// Build and insert \p Res = G_MERGE_VALUES \p Op0, ...
@@ -687,7 +731,7 @@
/// \pre The type of all \p Ops registers must be identical.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildMerge(const DstOp &Res, ArrayRef<unsigned> Ops);
+ MachineInstrBuilder buildMerge(const DstOp &Res, ArrayRef<Register> Ops);
/// Build and insert \p Res0, ... = G_UNMERGE_VALUES \p Op
///
@@ -700,7 +744,10 @@
///
/// \return a MachineInstrBuilder for the newly created instruction.
MachineInstrBuilder buildUnmerge(ArrayRef<LLT> Res, const SrcOp &Op);
- MachineInstrBuilder buildUnmerge(ArrayRef<unsigned> Res, const SrcOp &Op);
+ MachineInstrBuilder buildUnmerge(ArrayRef<Register> Res, const SrcOp &Op);
+
+ /// Build and insert an unmerge of \p Res sized pieces to cover \p Op
+ MachineInstrBuilder buildUnmerge(LLT Res, const SrcOp &Op);
/// Build and insert \p Res = G_BUILD_VECTOR \p Op0, ...
///
@@ -712,7 +759,7 @@
///
/// \return a MachineInstrBuilder for the newly created instruction.
MachineInstrBuilder buildBuildVector(const DstOp &Res,
- ArrayRef<unsigned> Ops);
+ ArrayRef<Register> Ops);
/// Build and insert \p Res = G_BUILD_VECTOR with \p Src replicated to fill
/// the number of elements
@@ -733,7 +780,7 @@
///
/// \return a MachineInstrBuilder for the newly created instruction.
MachineInstrBuilder buildBuildVectorTrunc(const DstOp &Res,
- ArrayRef<unsigned> Ops);
+ ArrayRef<Register> Ops);
/// Build and insert \p Res = G_CONCAT_VECTORS \p Op0, ...
///
@@ -747,10 +794,10 @@
///
/// \return a MachineInstrBuilder for the newly created instruction.
MachineInstrBuilder buildConcatVectors(const DstOp &Res,
- ArrayRef<unsigned> Ops);
+ ArrayRef<Register> Ops);
- MachineInstrBuilder buildInsert(unsigned Res, unsigned Src,
- unsigned Op, unsigned Index);
+ MachineInstrBuilder buildInsert(Register Res, Register Src,
+ Register Op, unsigned Index);
/// Build and insert either a G_INTRINSIC (if \p HasSideEffects is false) or
/// G_INTRINSIC_W_SIDE_EFFECTS instruction. Its first operand will be the
@@ -762,7 +809,9 @@
/// \pre setBasicBlock or setMI must have been called.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, unsigned Res,
+ MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef<Register> Res,
+ bool HasSideEffects);
+ MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef<DstOp> Res,
bool HasSideEffects);
/// Build and insert \p Res = G_FPTRUNC \p Op
@@ -877,8 +926,8 @@
///
/// \return a MachineInstrBuilder for the newly created instruction.
MachineInstrBuilder
- buildAtomicCmpXchgWithSuccess(unsigned OldValRes, unsigned SuccessRes,
- unsigned Addr, unsigned CmpVal, unsigned NewVal,
+ buildAtomicCmpXchgWithSuccess(Register OldValRes, Register SuccessRes,
+ Register Addr, Register CmpVal, Register NewVal,
MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMIC_CMPXCHG Addr, CmpVal, NewVal,
@@ -895,8 +944,8 @@
/// registers of the same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildAtomicCmpXchg(unsigned OldValRes, unsigned Addr,
- unsigned CmpVal, unsigned NewVal,
+ MachineInstrBuilder buildAtomicCmpXchg(Register OldValRes, Register Addr,
+ Register CmpVal, Register NewVal,
MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_<Opcode> Addr, Val, MMO`.
@@ -912,8 +961,8 @@
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildAtomicRMW(unsigned Opcode, unsigned OldValRes,
- unsigned Addr, unsigned Val,
+ MachineInstrBuilder buildAtomicRMW(unsigned Opcode, Register OldValRes,
+ Register Addr, Register Val,
MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_XCHG Addr, Val, MMO`.
@@ -928,8 +977,8 @@
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildAtomicRMWXchg(unsigned OldValRes, unsigned Addr,
- unsigned Val, MachineMemOperand &MMO);
+ MachineInstrBuilder buildAtomicRMWXchg(Register OldValRes, Register Addr,
+ Register Val, MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_ADD Addr, Val, MMO`.
///
@@ -943,8 +992,8 @@
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildAtomicRMWAdd(unsigned OldValRes, unsigned Addr,
- unsigned Val, MachineMemOperand &MMO);
+ MachineInstrBuilder buildAtomicRMWAdd(Register OldValRes, Register Addr,
+ Register Val, MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_SUB Addr, Val, MMO`.
///
@@ -958,8 +1007,8 @@
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildAtomicRMWSub(unsigned OldValRes, unsigned Addr,
- unsigned Val, MachineMemOperand &MMO);
+ MachineInstrBuilder buildAtomicRMWSub(Register OldValRes, Register Addr,
+ Register Val, MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_AND Addr, Val, MMO`.
///
@@ -973,8 +1022,8 @@
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildAtomicRMWAnd(unsigned OldValRes, unsigned Addr,
- unsigned Val, MachineMemOperand &MMO);
+ MachineInstrBuilder buildAtomicRMWAnd(Register OldValRes, Register Addr,
+ Register Val, MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_NAND Addr, Val, MMO`.
///
@@ -989,8 +1038,8 @@
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildAtomicRMWNand(unsigned OldValRes, unsigned Addr,
- unsigned Val, MachineMemOperand &MMO);
+ MachineInstrBuilder buildAtomicRMWNand(Register OldValRes, Register Addr,
+ Register Val, MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_OR Addr, Val, MMO`.
///
@@ -1004,8 +1053,8 @@
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildAtomicRMWOr(unsigned OldValRes, unsigned Addr,
- unsigned Val, MachineMemOperand &MMO);
+ MachineInstrBuilder buildAtomicRMWOr(Register OldValRes, Register Addr,
+ Register Val, MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_XOR Addr, Val, MMO`.
///
@@ -1019,8 +1068,8 @@
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildAtomicRMWXor(unsigned OldValRes, unsigned Addr,
- unsigned Val, MachineMemOperand &MMO);
+ MachineInstrBuilder buildAtomicRMWXor(Register OldValRes, Register Addr,
+ Register Val, MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_MAX Addr, Val, MMO`.
///
@@ -1035,8 +1084,8 @@
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildAtomicRMWMax(unsigned OldValRes, unsigned Addr,
- unsigned Val, MachineMemOperand &MMO);
+ MachineInstrBuilder buildAtomicRMWMax(Register OldValRes, Register Addr,
+ Register Val, MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_MIN Addr, Val, MMO`.
///
@@ -1051,8 +1100,8 @@
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildAtomicRMWMin(unsigned OldValRes, unsigned Addr,
- unsigned Val, MachineMemOperand &MMO);
+ MachineInstrBuilder buildAtomicRMWMin(Register OldValRes, Register Addr,
+ Register Val, MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_UMAX Addr, Val, MMO`.
///
@@ -1067,8 +1116,8 @@
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildAtomicRMWUmax(unsigned OldValRes, unsigned Addr,
- unsigned Val, MachineMemOperand &MMO);
+ MachineInstrBuilder buildAtomicRMWUmax(Register OldValRes, Register Addr,
+ Register Val, MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_UMIN Addr, Val, MMO`.
///
@@ -1083,8 +1132,11 @@
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
- MachineInstrBuilder buildAtomicRMWUmin(unsigned OldValRes, unsigned Addr,
- unsigned Val, MachineMemOperand &MMO);
+ MachineInstrBuilder buildAtomicRMWUmin(Register OldValRes, Register Addr,
+ Register Val, MachineMemOperand &MMO);
+
+ /// Build and insert `G_FENCE Ordering, Scope`.
+ MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope);
/// Build and insert \p Res = G_BLOCK_ADDR \p BA
///
@@ -1094,7 +1146,7 @@
/// \pre \p Res must be a generic virtual register of a pointer type.
///
/// \return The newly created instruction.
- MachineInstrBuilder buildBlockAddress(unsigned Res, const BlockAddress *BA);
+ MachineInstrBuilder buildBlockAddress(Register Res, const BlockAddress *BA);
/// Build and insert \p Res = G_ADD \p Op0, \p Op1
///
@@ -1207,6 +1259,131 @@
return buildInstr(TargetOpcode::G_OR, {Dst}, {Src0, Src1});
}
+ /// Build and insert \p Res = G_XOR \p Op0, \p Op1
+ MachineInstrBuilder buildXor(const DstOp &Dst, const SrcOp &Src0,
+ const SrcOp &Src1) {
+ return buildInstr(TargetOpcode::G_XOR, {Dst}, {Src0, Src1});
+ }
+
+ /// Build and insert a bitwise not,
+ /// \p NegOne = G_CONSTANT -1
+ /// \p Res = G_OR \p Op0, NegOne
+ MachineInstrBuilder buildNot(const DstOp &Dst, const SrcOp &Src0) {
+ auto NegOne = buildConstant(Dst.getLLTTy(*getMRI()), -1);
+ return buildInstr(TargetOpcode::G_XOR, {Dst}, {Src0, NegOne});
+ }
+
+ /// Build and insert \p Res = G_CTPOP \p Op0, \p Src0
+ MachineInstrBuilder buildCTPOP(const DstOp &Dst, const SrcOp &Src0) {
+ return buildInstr(TargetOpcode::G_CTPOP, {Dst}, {Src0});
+ }
+
+ /// Build and insert \p Res = G_CTLZ \p Op0, \p Src0
+ MachineInstrBuilder buildCTLZ(const DstOp &Dst, const SrcOp &Src0) {
+ return buildInstr(TargetOpcode::G_CTLZ, {Dst}, {Src0});
+ }
+
+ /// Build and insert \p Res = G_CTLZ_ZERO_UNDEF \p Op0, \p Src0
+ MachineInstrBuilder buildCTLZ_ZERO_UNDEF(const DstOp &Dst, const SrcOp &Src0) {
+ return buildInstr(TargetOpcode::G_CTLZ_ZERO_UNDEF, {Dst}, {Src0});
+ }
+
+ /// Build and insert \p Res = G_CTTZ \p Op0, \p Src0
+ MachineInstrBuilder buildCTTZ(const DstOp &Dst, const SrcOp &Src0) {
+ return buildInstr(TargetOpcode::G_CTTZ, {Dst}, {Src0});
+ }
+
+ /// Build and insert \p Res = G_CTTZ_ZERO_UNDEF \p Op0, \p Src0
+ MachineInstrBuilder buildCTTZ_ZERO_UNDEF(const DstOp &Dst, const SrcOp &Src0) {
+ return buildInstr(TargetOpcode::G_CTTZ_ZERO_UNDEF, {Dst}, {Src0});
+ }
+
+ /// Build and insert \p Res = G_FADD \p Op0, \p Op1
+ MachineInstrBuilder buildFAdd(const DstOp &Dst, const SrcOp &Src0,
+ const SrcOp &Src1) {
+ return buildInstr(TargetOpcode::G_FADD, {Dst}, {Src0, Src1});
+ }
+
+ /// Build and insert \p Res = G_FSUB \p Op0, \p Op1
+ MachineInstrBuilder buildFSub(const DstOp &Dst, const SrcOp &Src0,
+ const SrcOp &Src1) {
+ return buildInstr(TargetOpcode::G_FSUB, {Dst}, {Src0, Src1});
+ }
+
+ /// Build and insert \p Res = G_FMA \p Op0, \p Op1, \p Op2
+ MachineInstrBuilder buildFMA(const DstOp &Dst, const SrcOp &Src0,
+ const SrcOp &Src1, const SrcOp &Src2) {
+ return buildInstr(TargetOpcode::G_FMA, {Dst}, {Src0, Src1, Src2});
+ }
+
+ /// Build and insert \p Res = G_FNEG \p Op0
+ MachineInstrBuilder buildFNeg(const DstOp &Dst, const SrcOp &Src0) {
+ return buildInstr(TargetOpcode::G_FNEG, {Dst}, {Src0});
+ }
+
+ /// Build and insert \p Res = G_FABS \p Op0
+ MachineInstrBuilder buildFAbs(const DstOp &Dst, const SrcOp &Src0) {
+ return buildInstr(TargetOpcode::G_FABS, {Dst}, {Src0});
+ }
+
+ /// Build and insert \p Res = G_FCOPYSIGN \p Op0, \p Op1
+ MachineInstrBuilder buildFCopysign(const DstOp &Dst, const SrcOp &Src0,
+ const SrcOp &Src1) {
+ return buildInstr(TargetOpcode::G_FCOPYSIGN, {Dst}, {Src0, Src1});
+ }
+
+ /// Build and insert \p Res = G_UITOFP \p Src0
+ MachineInstrBuilder buildUITOFP(const DstOp &Dst, const SrcOp &Src0) {
+ return buildInstr(TargetOpcode::G_UITOFP, {Dst}, {Src0});
+ }
+
+ /// Build and insert \p Res = G_SITOFP \p Src0
+ MachineInstrBuilder buildSITOFP(const DstOp &Dst, const SrcOp &Src0) {
+ return buildInstr(TargetOpcode::G_SITOFP, {Dst}, {Src0});
+ }
+
+ /// Build and insert \p Res = G_FPTOUI \p Src0
+ MachineInstrBuilder buildFPTOUI(const DstOp &Dst, const SrcOp &Src0) {
+ return buildInstr(TargetOpcode::G_FPTOUI, {Dst}, {Src0});
+ }
+
+ /// Build and insert \p Res = G_FPTOSI \p Src0
+ MachineInstrBuilder buildFPTOSI(const DstOp &Dst, const SrcOp &Src0) {
+ return buildInstr(TargetOpcode::G_FPTOSI, {Dst}, {Src0});
+ }
+
+ /// Build and insert \p Res = G_SMIN \p Op0, \p Op1
+ MachineInstrBuilder buildSMin(const DstOp &Dst, const SrcOp &Src0,
+ const SrcOp &Src1) {
+ return buildInstr(TargetOpcode::G_SMIN, {Dst}, {Src0, Src1});
+ }
+
+ /// Build and insert \p Res = G_SMAX \p Op0, \p Op1
+ MachineInstrBuilder buildSMax(const DstOp &Dst, const SrcOp &Src0,
+ const SrcOp &Src1) {
+ return buildInstr(TargetOpcode::G_SMAX, {Dst}, {Src0, Src1});
+ }
+
+ /// Build and insert \p Res = G_UMIN \p Op0, \p Op1
+ MachineInstrBuilder buildUMin(const DstOp &Dst, const SrcOp &Src0,
+ const SrcOp &Src1) {
+ return buildInstr(TargetOpcode::G_UMIN, {Dst}, {Src0, Src1});
+ }
+
+ /// Build and insert \p Res = G_UMAX \p Op0, \p Op1
+ MachineInstrBuilder buildUMax(const DstOp &Dst, const SrcOp &Src0,
+ const SrcOp &Src1) {
+ return buildInstr(TargetOpcode::G_UMAX, {Dst}, {Src0, Src1});
+ }
+
+ /// Build and insert \p Res = G_JUMP_TABLE \p JTI
+ ///
+ /// G_JUMP_TABLE sets \p Res to the address of the jump table specified by
+ /// the jump table index \p JTI.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildJumpTable(const LLT PtrTy, unsigned JTI);
+
virtual MachineInstrBuilder buildInstr(unsigned Opc, ArrayRef<DstOp> DstOps,
ArrayRef<SrcOp> SrcOps,
Optional<unsigned> Flags = None);
diff --git a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/RegBankSelect.h b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/RegBankSelect.h
index c31ca5c..d9d076b 100644
--- a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/RegBankSelect.h
+++ b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/RegBankSelect.h
@@ -523,7 +523,7 @@
/// \p OnlyAssign == true means that \p Reg just needs to be assigned a
/// register bank. I.e., no repairing is necessary to have the
/// assignment match.
- bool assignmentMatch(unsigned Reg,
+ bool assignmentMatch(Register Reg,
const RegisterBankInfo::ValueMapping &ValMapping,
bool &OnlyAssign) const;
@@ -562,7 +562,7 @@
bool repairReg(MachineOperand &MO,
const RegisterBankInfo::ValueMapping &ValMapping,
RegBankSelect::RepairingPlacement &RepairPt,
- const iterator_range<SmallVectorImpl<unsigned>::const_iterator>
+ const iterator_range<SmallVectorImpl<Register>::const_iterator>
&NewVRegs);
/// Return the cost of the instruction needed to map \p MO to \p ValMapping.
@@ -633,6 +633,11 @@
MachineFunctionProperties::Property::RegBankSelected);
}
+ MachineFunctionProperties getClearedProperties() const override {
+ return MachineFunctionProperties()
+ .set(MachineFunctionProperties::Property::NoPHIs);
+ }
+
/// Walk through \p MF and assign a register bank to every virtual register
/// that are still mapped to nothing.
/// The target needs to provide a RegisterBankInfo and in particular
diff --git a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h
index 37a63d1..e84b1c3 100644
--- a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h
+++ b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h
@@ -18,6 +18,7 @@
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/iterator_range.h"
+#include "llvm/CodeGen/Register.h"
#include "llvm/Support/ErrorHandling.h"
#include <cassert>
#include <initializer_list>
@@ -193,7 +194,7 @@
unsigned Cost = 0;
/// Mapping of all the operands.
- const ValueMapping *OperandsMapping;
+ const ValueMapping *OperandsMapping = nullptr;
/// Number of operands.
unsigned NumOperands = 0;
@@ -210,15 +211,11 @@
/// The rationale is that it is more efficient for the optimizers
/// to be able to assume that the mapping of the ith operand is
/// at the index i.
- ///
- /// \pre ID != InvalidMappingID
InstructionMapping(unsigned ID, unsigned Cost,
const ValueMapping *OperandsMapping,
unsigned NumOperands)
: ID(ID), Cost(Cost), OperandsMapping(OperandsMapping),
NumOperands(NumOperands) {
- assert(getID() != InvalidMappingID &&
- "Use the default constructor for invalid mapping");
}
/// Default constructor.
@@ -285,7 +282,7 @@
SmallVector<int, 8> OpToNewVRegIdx;
/// Hold the registers that will be used to map MI with InstrMapping.
- SmallVector<unsigned, 8> NewVRegs;
+ SmallVector<Register, 8> NewVRegs;
/// Current MachineRegisterInfo, used to create new virtual registers.
MachineRegisterInfo &MRI;
@@ -306,15 +303,15 @@
/// \return The iterator range for the space created.
//
/// \pre getMI().getOperand(OpIdx).isReg()
- iterator_range<SmallVectorImpl<unsigned>::iterator>
+ iterator_range<SmallVectorImpl<Register>::iterator>
getVRegsMem(unsigned OpIdx);
/// Get the end iterator for a range starting at \p StartIdx and
/// spannig \p NumVal in NewVRegs.
/// \pre StartIdx + NumVal <= NewVRegs.size()
- SmallVectorImpl<unsigned>::const_iterator
+ SmallVectorImpl<Register>::const_iterator
getNewVRegsEnd(unsigned StartIdx, unsigned NumVal) const;
- SmallVectorImpl<unsigned>::iterator getNewVRegsEnd(unsigned StartIdx,
+ SmallVectorImpl<Register>::iterator getNewVRegsEnd(unsigned StartIdx,
unsigned NumVal);
public:
@@ -360,7 +357,7 @@
///
/// \post the \p PartialMapIdx-th register of the value mapping of the \p
/// OpIdx-th operand has been set.
- void setVRegs(unsigned OpIdx, unsigned PartialMapIdx, unsigned NewVReg);
+ void setVRegs(unsigned OpIdx, unsigned PartialMapIdx, Register NewVReg);
/// Get all the virtual registers required to map the \p OpIdx-th operand of
/// the instruction.
@@ -374,7 +371,7 @@
///
/// \pre getMI().getOperand(OpIdx).isReg()
/// \pre ForDebug || All partial mappings have been set a register
- iterator_range<SmallVectorImpl<unsigned>::const_iterator>
+ iterator_range<SmallVectorImpl<Register>::const_iterator>
getVRegs(unsigned OpIdx, bool ForDebug = false) const;
/// Print this operands mapper on dbgs() stream.
@@ -438,7 +435,7 @@
/// Get the MinimalPhysRegClass for Reg.
/// \pre Reg is a physical register.
const TargetRegisterClass &
- getMinimalPhysRegClass(unsigned Reg, const TargetRegisterInfo &TRI) const;
+ getMinimalPhysRegClass(Register Reg, const TargetRegisterInfo &TRI) const;
/// Try to get the mapping of \p MI.
/// See getInstrMapping for more details on what a mapping represents.
@@ -583,7 +580,7 @@
/// or a register bank, then this returns nullptr.
///
/// \pre Reg != 0 (NoRegister)
- const RegisterBank *getRegBank(unsigned Reg, const MachineRegisterInfo &MRI,
+ const RegisterBank *getRegBank(Register Reg, const MachineRegisterInfo &MRI,
const TargetRegisterInfo &TRI) const;
/// Get the total number of register banks.
@@ -621,6 +618,12 @@
return &A != &B;
}
+ /// \returns true if emitting a copy from \p Src to \p Dst is impossible.
+ bool cannotCopy(const RegisterBank &Dst, const RegisterBank &Src,
+ unsigned Size) const {
+ return copyCost(Dst, Src, Size) == std::numeric_limits<unsigned>::max();
+ }
+
/// Get the cost of using \p ValMapping to decompose a register. This is
/// similar to ::copyCost, except for cases where multiple copy-like
/// operations need to be inserted. If the register is used as a source
@@ -638,7 +641,7 @@
/// \note Use MachineRegisterInfo::constrainRegAttrs instead for any non-isel
/// purpose, including non-select passes of GlobalISel
static const TargetRegisterClass *
- constrainGenericRegister(unsigned Reg, const TargetRegisterClass &RC,
+ constrainGenericRegister(Register Reg, const TargetRegisterClass &RC,
MachineRegisterInfo &MRI);
/// Identifier used when the related instruction mapping instance
@@ -723,7 +726,7 @@
/// virtual register.
///
/// \pre \p Reg != 0 (NoRegister).
- unsigned getSizeInBits(unsigned Reg, const MachineRegisterInfo &MRI,
+ unsigned getSizeInBits(Register Reg, const MachineRegisterInfo &MRI,
const TargetRegisterInfo &TRI) const;
/// Check that information hold by this instance make sense for the
diff --git a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/Utils.h b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/Utils.h
index dd14a0c..6946aad 100644
--- a/linux-x64/clang/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/linux-x64/clang/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -15,6 +15,7 @@
#define LLVM_CODEGEN_GLOBALISEL_UTILS_H
#include "llvm/ADT/StringRef.h"
+#include "llvm/CodeGen/Register.h"
namespace llvm {
@@ -36,21 +37,37 @@
class APFloat;
/// Try to constrain Reg to the specified register class. If this fails,
-/// create a new virtual register in the correct class and insert a COPY before
-/// \p InsertPt. The debug location of \p InsertPt is used for the new copy.
+/// create a new virtual register in the correct class.
///
/// \return The virtual register constrained to the right register class.
unsigned constrainRegToClass(MachineRegisterInfo &MRI,
const TargetInstrInfo &TII,
- const RegisterBankInfo &RBI,
- MachineInstr &InsertPt, unsigned Reg,
+ const RegisterBankInfo &RBI, unsigned Reg,
const TargetRegisterClass &RegClass);
+/// Constrain the Register operand OpIdx, so that it is now constrained to the
+/// TargetRegisterClass passed as an argument (RegClass).
+/// If this fails, create a new virtual register in the correct class and
+/// insert a COPY before \p InsertPt if it is a use or after if it is a
+/// definition. The debug location of \p InsertPt is used for the new copy.
+///
+/// \return The virtual register constrained to the right register class.
+unsigned constrainOperandRegClass(const MachineFunction &MF,
+ const TargetRegisterInfo &TRI,
+ MachineRegisterInfo &MRI,
+ const TargetInstrInfo &TII,
+ const RegisterBankInfo &RBI,
+ MachineInstr &InsertPt,
+ const TargetRegisterClass &RegClass,
+ const MachineOperand &RegMO, unsigned OpIdx);
+
/// Try to constrain Reg so that it is usable by argument OpIdx of the
/// provided MCInstrDesc \p II. If this fails, create a new virtual
-/// register in the correct class and insert a COPY before \p InsertPt.
-/// This is equivalent to constrainRegToClass() with RegClass obtained from the
-/// MCInstrDesc. The debug location of \p InsertPt is used for the new copy.
+/// register in the correct class and insert a COPY before \p InsertPt
+/// if it is a use or after if it is a definition.
+/// This is equivalent to constrainOperandRegClass(..., RegClass, ...)
+/// with RegClass obtained from the MCInstrDesc. The debug location of \p
+/// InsertPt is used for the new copy.
///
/// \return The virtual register constrained to the right register class.
unsigned constrainOperandRegClass(const MachineFunction &MF,
@@ -89,17 +106,40 @@
const char *PassName, StringRef Msg,
const MachineInstr &MI);
+/// If \p VReg is defined by a G_CONSTANT fits in int64_t
+/// returns it.
Optional<int64_t> getConstantVRegVal(unsigned VReg,
const MachineRegisterInfo &MRI);
+/// Simple struct used to hold a constant integer value and a virtual
+/// register.
+struct ValueAndVReg {
+ int64_t Value;
+ unsigned VReg;
+};
+/// If \p VReg is defined by a statically evaluable chain of
+/// instructions rooted on a G_CONSTANT (\p LookThroughInstrs == true)
+/// and that constant fits in int64_t, returns its value as well as
+/// the virtual register defined by this G_CONSTANT.
+/// When \p LookThroughInstrs == false, this function behaves like
+/// getConstantVRegVal.
+Optional<ValueAndVReg>
+getConstantVRegValWithLookThrough(unsigned VReg, const MachineRegisterInfo &MRI,
+ bool LookThroughInstrs = true);
const ConstantFP* getConstantFPVRegVal(unsigned VReg,
const MachineRegisterInfo &MRI);
/// See if Reg is defined by an single def instruction that is
/// Opcode. Also try to do trivial folding if it's a COPY with
/// same types. Returns null otherwise.
-MachineInstr *getOpcodeDef(unsigned Opcode, unsigned Reg,
+MachineInstr *getOpcodeDef(unsigned Opcode, Register Reg,
const MachineRegisterInfo &MRI);
+/// Find the def instruction for \p Reg, folding away any trivial copies. Note
+/// it may still return a COPY, if it changes the type. May return nullptr if \p
+/// Reg is not a generic virtual register.
+MachineInstr *getDefIgnoringCopies(Register Reg,
+ const MachineRegisterInfo &MRI);
+
/// Returns an APFloat from Val converted to the appropriate size.
APFloat getAPFloatFromSize(double Val, unsigned Size);
diff --git a/linux-x64/clang/include/llvm/CodeGen/ISDOpcodes.h b/linux-x64/clang/include/llvm/CodeGen/ISDOpcodes.h
index ed4bfe7..acf27dc 100644
--- a/linux-x64/clang/include/llvm/CodeGen/ISDOpcodes.h
+++ b/linux-x64/clang/include/llvm/CodeGen/ISDOpcodes.h
@@ -278,6 +278,11 @@
/// multiplication on 2 integers.
SMULFIX, UMULFIX,
+ /// Same as the corresponding unsaturated fixed point instructions, but the
+ /// result is clamped between the min and max values representable by the
+ /// bits of the first 2 operands.
+ SMULFIXSAT,
+
/// Simple binary floating point operators.
FADD, FSUB, FMUL, FDIV, FREM,
@@ -297,6 +302,26 @@
STRICT_FRINT, STRICT_FNEARBYINT, STRICT_FMAXNUM, STRICT_FMINNUM,
STRICT_FCEIL, STRICT_FFLOOR, STRICT_FROUND, STRICT_FTRUNC,
+ /// X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating
+ /// point type down to the precision of the destination VT. TRUNC is a
+ /// flag, which is always an integer that is zero or one. If TRUNC is 0,
+ /// this is a normal rounding, if it is 1, this FP_ROUND is known to not
+ /// change the value of Y.
+ ///
+ /// The TRUNC = 1 case is used in cases where we know that the value will
+ /// not be modified by the node, because Y is not using any of the extra
+ /// precision of source type. This allows certain transformations like
+ /// STRICT_FP_EXTEND(STRICT_FP_ROUND(X,1)) -> X which are not safe for
+ /// STRICT_FP_EXTEND(STRICT_FP_ROUND(X,0)) because the extra bits aren't
+ /// removed.
+ /// It is used to limit optimizations while the DAG is being optimized.
+ STRICT_FP_ROUND,
+
+ /// X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP
+ /// type.
+ /// It is used to limit optimizations while the DAG is being optimized.
+ STRICT_FP_EXTEND,
+
/// FMA - Perform a * b + c with no intermediate rounding step.
FMA,
@@ -585,6 +610,8 @@
FNEG, FABS, FSQRT, FCBRT, FSIN, FCOS, FPOWI, FPOW,
FLOG, FLOG2, FLOG10, FEXP, FEXP2,
FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR,
+ LROUND, LLROUND, LRINT, LLRINT,
+
/// FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two
/// values.
//
@@ -872,11 +899,14 @@
VECREDUCE_STRICT_FADD, VECREDUCE_STRICT_FMUL,
/// These reductions are non-strict, and have a single vector operand.
VECREDUCE_FADD, VECREDUCE_FMUL,
+ /// FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
+ VECREDUCE_FMAX, VECREDUCE_FMIN,
+ /// Integer reductions may have a result type larger than the vector element
+ /// type. However, the reduction is performed using the vector element type
+ /// and the value in the top bits is unspecified.
VECREDUCE_ADD, VECREDUCE_MUL,
VECREDUCE_AND, VECREDUCE_OR, VECREDUCE_XOR,
VECREDUCE_SMAX, VECREDUCE_SMIN, VECREDUCE_UMAX, VECREDUCE_UMIN,
- /// FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
- VECREDUCE_FMAX, VECREDUCE_FMIN,
/// BUILTIN_OP_END - This must be the last enum value in this list.
/// The target-specific pre-isel opcode values start here.
diff --git a/linux-x64/clang/include/llvm/CodeGen/LiveInterval.h b/linux-x64/clang/include/llvm/CodeGen/LiveInterval.h
index 622c124..8bb8816 100644
--- a/linux-x64/clang/include/llvm/CodeGen/LiveInterval.h
+++ b/linux-x64/clang/include/llvm/CodeGen/LiveInterval.h
@@ -605,6 +605,44 @@
/// activated in the constructor of the live range.
void flushSegmentSet();
+ /// Stores indexes from the input index sequence R at which this LiveRange
+ /// is live to the output O iterator.
+ /// R is a range of _ascending sorted_ _random_ access iterators
+ /// to the input indexes. Indexes stored at O are ascending sorted so it
+ /// can be used directly in the subsequent search (for example for
+ /// subranges). Returns true if found at least one index.
+ template <typename Range, typename OutputIt>
+ bool findIndexesLiveAt(Range &&R, OutputIt O) const {
+ assert(std::is_sorted(R.begin(), R.end()));
+ auto Idx = R.begin(), EndIdx = R.end();
+ auto Seg = segments.begin(), EndSeg = segments.end();
+ bool Found = false;
+ while (Idx != EndIdx && Seg != EndSeg) {
+ // if the Seg is lower find first segment that is above Idx using binary
+ // search
+ if (Seg->end <= *Idx) {
+ Seg = std::upper_bound(++Seg, EndSeg, *Idx,
+ [=](typename std::remove_reference<decltype(*Idx)>::type V,
+ const typename std::remove_reference<decltype(*Seg)>::type &S) {
+ return V < S.end;
+ });
+ if (Seg == EndSeg)
+ break;
+ }
+ auto NotLessStart = std::lower_bound(Idx, EndIdx, Seg->start);
+ if (NotLessStart == EndIdx)
+ break;
+ auto NotLessEnd = std::lower_bound(NotLessStart, EndIdx, Seg->end);
+ if (NotLessEnd != NotLessStart) {
+ Found = true;
+ O = std::copy(NotLessStart, NotLessEnd, O);
+ }
+ Idx = NotLessEnd;
+ ++Seg;
+ }
+ return Found;
+ }
+
void print(raw_ostream &OS) const;
void dump() const;
@@ -789,8 +827,15 @@
/// L000F, refining for mask L0018. Will split the L00F0 lane into
/// L00E0 and L0010 and the L000F lane into L0007 and L0008. The Mod
/// function will be applied to the L0010 and L0008 subranges.
+ ///
+ /// \p Indexes and \p TRI are required to clean up the VNIs that
+ /// don't defne the related lane masks after they get shrunk. E.g.,
+ /// when L000F gets split into L0007 and L0008 maybe only a subset
+ /// of the VNIs that defined L000F defines L0007.
void refineSubRanges(BumpPtrAllocator &Allocator, LaneBitmask LaneMask,
- std::function<void(LiveInterval::SubRange&)> Apply);
+ std::function<void(LiveInterval::SubRange &)> Apply,
+ const SlotIndexes &Indexes,
+ const TargetRegisterInfo &TRI);
bool operator<(const LiveInterval& other) const {
const SlotIndex &thisIndex = beginIndex();
diff --git a/linux-x64/clang/include/llvm/CodeGen/LiveIntervals.h b/linux-x64/clang/include/llvm/CodeGen/LiveIntervals.h
index d0791be..588b0f9 100644
--- a/linux-x64/clang/include/llvm/CodeGen/LiveIntervals.h
+++ b/linux-x64/clang/include/llvm/CodeGen/LiveIntervals.h
@@ -417,6 +417,15 @@
RegUnitRanges[Unit] = nullptr;
}
+ /// Remove associated live ranges for the register units associated with \p
+ /// Reg. Subsequent uses should rely on on-demand recomputation. \note This
+ /// method can result in inconsistent liveness tracking if multiple phyical
+ /// registers share a regunit, and should be used cautiously.
+ void removeAllRegUnitsForPhysReg(unsigned Reg) {
+ for (MCRegUnitIterator Units(Reg, TRI); Units.isValid(); ++Units)
+ removeRegUnit(*Units);
+ }
+
/// Remove value numbers and related live segments starting at position
/// \p Pos that are part of any liverange of physical register \p Reg or one
/// of its subregisters.
diff --git a/linux-x64/clang/include/llvm/CodeGen/MIRParser/MIParser.h b/linux-x64/clang/include/llvm/CodeGen/MIRParser/MIParser.h
new file mode 100644
index 0000000..4e32a04
--- /dev/null
+++ b/linux-x64/clang/include/llvm/CodeGen/MIRParser/MIParser.h
@@ -0,0 +1,233 @@
+//===- MIParser.h - Machine Instructions Parser -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the function that parses the machine instructions.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_CODEGEN_MIRPARSER_MIPARSER_H
+#define LLVM_LIB_CODEGEN_MIRPARSER_MIPARSER_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/Support/Allocator.h"
+
+namespace llvm {
+
+class MachineBasicBlock;
+class MachineFunction;
+class MDNode;
+class RegisterBank;
+struct SlotMapping;
+class SMDiagnostic;
+class SourceMgr;
+class StringRef;
+class TargetRegisterClass;
+class TargetSubtargetInfo;
+
+struct VRegInfo {
+ enum uint8_t {
+ UNKNOWN, NORMAL, GENERIC, REGBANK
+ } Kind = UNKNOWN;
+ bool Explicit = false; ///< VReg was explicitly specified in the .mir file.
+ union {
+ const TargetRegisterClass *RC;
+ const RegisterBank *RegBank;
+ } D;
+ unsigned VReg;
+ unsigned PreferredReg = 0;
+};
+
+using Name2RegClassMap = StringMap<const TargetRegisterClass *>;
+using Name2RegBankMap = StringMap<const RegisterBank *>;
+
+struct PerTargetMIParsingState {
+private:
+ const TargetSubtargetInfo &Subtarget;
+
+ /// Maps from instruction names to op codes.
+ StringMap<unsigned> Names2InstrOpCodes;
+
+ /// Maps from register names to registers.
+ StringMap<unsigned> Names2Regs;
+
+ /// Maps from register mask names to register masks.
+ StringMap<const uint32_t *> Names2RegMasks;
+
+ /// Maps from subregister names to subregister indices.
+ StringMap<unsigned> Names2SubRegIndices;
+
+ /// Maps from target index names to target indices.
+ StringMap<int> Names2TargetIndices;
+
+ /// Maps from direct target flag names to the direct target flag values.
+ StringMap<unsigned> Names2DirectTargetFlags;
+
+ /// Maps from direct target flag names to the bitmask target flag values.
+ StringMap<unsigned> Names2BitmaskTargetFlags;
+
+ /// Maps from MMO target flag names to MMO target flag values.
+ StringMap<MachineMemOperand::Flags> Names2MMOTargetFlags;
+
+ /// Maps from register class names to register classes.
+ Name2RegClassMap Names2RegClasses;
+
+ /// Maps from register bank names to register banks.
+ Name2RegBankMap Names2RegBanks;
+
+ void initNames2InstrOpCodes();
+ void initNames2Regs();
+ void initNames2RegMasks();
+ void initNames2SubRegIndices();
+ void initNames2TargetIndices();
+ void initNames2DirectTargetFlags();
+ void initNames2BitmaskTargetFlags();
+ void initNames2MMOTargetFlags();
+
+ void initNames2RegClasses();
+ void initNames2RegBanks();
+
+public:
+ /// Try to convert an instruction name to an opcode. Return true if the
+ /// instruction name is invalid.
+ bool parseInstrName(StringRef InstrName, unsigned &OpCode);
+
+ /// Try to convert a register name to a register number. Return true if the
+ /// register name is invalid.
+ bool getRegisterByName(StringRef RegName, unsigned &Reg);
+
+ /// Check if the given identifier is a name of a register mask.
+ ///
+ /// Return null if the identifier isn't a register mask.
+ const uint32_t *getRegMask(StringRef Identifier);
+
+ /// Check if the given identifier is a name of a subregister index.
+ ///
+ /// Return 0 if the name isn't a subregister index class.
+ unsigned getSubRegIndex(StringRef Name);
+
+ /// Try to convert a name of target index to the corresponding target index.
+ ///
+ /// Return true if the name isn't a name of a target index.
+ bool getTargetIndex(StringRef Name, int &Index);
+
+ /// Try to convert a name of a direct target flag to the corresponding
+ /// target flag.
+ ///
+ /// Return true if the name isn't a name of a direct flag.
+ bool getDirectTargetFlag(StringRef Name, unsigned &Flag);
+
+ /// Try to convert a name of a bitmask target flag to the corresponding
+ /// target flag.
+ ///
+ /// Return true if the name isn't a name of a bitmask target flag.
+ bool getBitmaskTargetFlag(StringRef Name, unsigned &Flag);
+
+ /// Try to convert a name of a MachineMemOperand target flag to the
+ /// corresponding target flag.
+ ///
+ /// Return true if the name isn't a name of a target MMO flag.
+ bool getMMOTargetFlag(StringRef Name, MachineMemOperand::Flags &Flag);
+
+ /// Check if the given identifier is a name of a register class.
+ ///
+ /// Return null if the name isn't a register class.
+ const TargetRegisterClass *getRegClass(StringRef Name);
+
+ /// Check if the given identifier is a name of a register bank.
+ ///
+ /// Return null if the name isn't a register bank.
+ const RegisterBank *getRegBank(StringRef Name);
+
+ PerTargetMIParsingState(const TargetSubtargetInfo &STI)
+ : Subtarget(STI) {
+ initNames2RegClasses();
+ initNames2RegBanks();
+ }
+
+ ~PerTargetMIParsingState() = default;
+
+ void setTarget(const TargetSubtargetInfo &NewSubtarget);
+};
+
+struct PerFunctionMIParsingState {
+ BumpPtrAllocator Allocator;
+ MachineFunction &MF;
+ SourceMgr *SM;
+ const SlotMapping &IRSlots;
+ PerTargetMIParsingState &Target;
+
+ DenseMap<unsigned, MachineBasicBlock *> MBBSlots;
+ DenseMap<unsigned, VRegInfo *> VRegInfos;
+ StringMap<VRegInfo *> VRegInfosNamed;
+ DenseMap<unsigned, int> FixedStackObjectSlots;
+ DenseMap<unsigned, int> StackObjectSlots;
+ DenseMap<unsigned, unsigned> ConstantPoolSlots;
+ DenseMap<unsigned, unsigned> JumpTableSlots;
+
+ PerFunctionMIParsingState(MachineFunction &MF, SourceMgr &SM,
+ const SlotMapping &IRSlots,
+ PerTargetMIParsingState &Target);
+
+ VRegInfo &getVRegInfo(unsigned Num);
+ VRegInfo &getVRegInfoNamed(StringRef RegName);
+};
+
+/// Parse the machine basic block definitions, and skip the machine
+/// instructions.
+///
+/// This function runs the first parsing pass on the machine function's body.
+/// It parses only the machine basic block definitions and creates the machine
+/// basic blocks in the given machine function.
+///
+/// The machine instructions aren't parsed during the first pass because all
+/// the machine basic blocks aren't defined yet - this makes it impossible to
+/// resolve the machine basic block references.
+///
+/// Return true if an error occurred.
+bool parseMachineBasicBlockDefinitions(PerFunctionMIParsingState &PFS,
+ StringRef Src, SMDiagnostic &Error);
+
+/// Parse the machine instructions.
+///
+/// This function runs the second parsing pass on the machine function's body.
+/// It skips the machine basic block definitions and parses only the machine
+/// instructions and basic block attributes like liveins and successors.
+///
+/// The second parsing pass assumes that the first parsing pass already ran
+/// on the given source string.
+///
+/// Return true if an error occurred.
+bool parseMachineInstructions(PerFunctionMIParsingState &PFS, StringRef Src,
+ SMDiagnostic &Error);
+
+bool parseMBBReference(PerFunctionMIParsingState &PFS,
+ MachineBasicBlock *&MBB, StringRef Src,
+ SMDiagnostic &Error);
+
+bool parseRegisterReference(PerFunctionMIParsingState &PFS,
+ unsigned &Reg, StringRef Src,
+ SMDiagnostic &Error);
+
+bool parseNamedRegisterReference(PerFunctionMIParsingState &PFS, unsigned &Reg,
+ StringRef Src, SMDiagnostic &Error);
+
+bool parseVirtualRegisterReference(PerFunctionMIParsingState &PFS,
+ VRegInfo *&Info, StringRef Src,
+ SMDiagnostic &Error);
+
+bool parseStackObjectReference(PerFunctionMIParsingState &PFS, int &FI,
+ StringRef Src, SMDiagnostic &Error);
+
+bool parseMDNode(PerFunctionMIParsingState &PFS, MDNode *&Node, StringRef Src,
+ SMDiagnostic &Error);
+
+} // end namespace llvm
+
+#endif // LLVM_LIB_CODEGEN_MIRPARSER_MIPARSER_H
diff --git a/linux-x64/clang/include/llvm/CodeGen/MIRPrinter.h b/linux-x64/clang/include/llvm/CodeGen/MIRPrinter.h
index b95cf0f..a4b03a7 100644
--- a/linux-x64/clang/include/llvm/CodeGen/MIRPrinter.h
+++ b/linux-x64/clang/include/llvm/CodeGen/MIRPrinter.h
@@ -1,4 +1,4 @@
-//===- MIRPrinter.h - MIR serialization format printer --------------------===//
+//===- MIRPrinter.h - MIR serialization format printer ----------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
diff --git a/linux-x64/clang/include/llvm/CodeGen/MIRYamlMapping.h b/linux-x64/clang/include/llvm/CodeGen/MIRYamlMapping.h
index b600ec9..94e76a7 100644
--- a/linux-x64/clang/include/llvm/CodeGen/MIRYamlMapping.h
+++ b/linux-x64/clang/include/llvm/CodeGen/MIRYamlMapping.h
@@ -1,4 +1,4 @@
-//===- MIRYamlMapping.h - Describes the mapping between MIR and YAML ------===//
+//===- MIRYamlMapping.h - Describe mapping between MIR and YAML--*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -17,6 +17,7 @@
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
+#include "llvm/CodeGen/TargetFrameLowering.h"
#include "llvm/Support/SMLoc.h"
#include "llvm/Support/YAMLTraits.h"
#include "llvm/Support/raw_ostream.h"
@@ -36,6 +37,7 @@
StringValue() = default;
StringValue(std::string Value) : Value(std::move(Value)) {}
+ StringValue(const char Val[]) : Value(Val) {}
bool operator==(const StringValue &Other) const {
return Value == Other.Value;
@@ -211,7 +213,7 @@
int64_t Offset = 0;
uint64_t Size = 0;
unsigned Alignment = 0;
- uint8_t StackID = 0;
+ TargetStackID::Value StackID;
StringValue CalleeSavedRegister;
bool CalleeSavedRestored = true;
Optional<int64_t> LocalOffset;
@@ -251,7 +253,7 @@
if (Object.Type != MachineStackObject::VariableSized)
YamlIO.mapRequired("size", Object.Size);
YamlIO.mapOptional("alignment", Object.Alignment, (unsigned)0);
- YamlIO.mapOptional("stack-id", Object.StackID);
+ YamlIO.mapOptional("stack-id", Object.StackID, TargetStackID::Default);
YamlIO.mapOptional("callee-saved-register", Object.CalleeSavedRegister,
StringValue()); // Don't print it out when it's empty.
YamlIO.mapOptional("callee-saved-restored", Object.CalleeSavedRestored,
@@ -277,7 +279,7 @@
int64_t Offset = 0;
uint64_t Size = 0;
unsigned Alignment = 0;
- uint8_t StackID = 0;
+ TargetStackID::Value StackID;
bool IsImmutable = false;
bool IsAliased = false;
StringValue CalleeSavedRegister;
@@ -307,6 +309,15 @@
}
};
+template <>
+struct ScalarEnumerationTraits<TargetStackID::Value> {
+ static void enumeration(yaml::IO &IO, TargetStackID::Value &ID) {
+ IO.enumCase(ID, "default", TargetStackID::Default);
+ IO.enumCase(ID, "sgpr-spill", TargetStackID::SGPRSpill);
+ IO.enumCase(ID, "noalloc", TargetStackID::NoAlloc);
+ }
+};
+
template <> struct MappingTraits<FixedMachineStackObject> {
static void mapping(yaml::IO &YamlIO, FixedMachineStackObject &Object) {
YamlIO.mapRequired("id", Object.ID);
@@ -316,7 +327,7 @@
YamlIO.mapOptional("offset", Object.Offset, (int64_t)0);
YamlIO.mapOptional("size", Object.Size, (uint64_t)0);
YamlIO.mapOptional("alignment", Object.Alignment, (unsigned)0);
- YamlIO.mapOptional("stack-id", Object.StackID);
+ YamlIO.mapOptional("stack-id", Object.StackID, TargetStackID::Default);
if (Object.Type != FixedMachineStackObject::SpillSlot) {
YamlIO.mapOptional("isImmutable", Object.IsImmutable, false);
YamlIO.mapOptional("isAliased", Object.IsAliased, false);
@@ -336,6 +347,66 @@
static const bool flow = true;
};
+
+/// Serializable representation of CallSiteInfo.
+struct CallSiteInfo {
+ // Representation of call argument and register which is used to
+ // transfer it.
+ struct ArgRegPair {
+ StringValue Reg;
+ uint16_t ArgNo;
+
+ bool operator==(const ArgRegPair &Other) const {
+ return Reg == Other.Reg && ArgNo == Other.ArgNo;
+ }
+ };
+
+ /// Identifies call instruction location in machine function.
+ struct MachineInstrLoc {
+ unsigned BlockNum;
+ unsigned Offset;
+
+ bool operator==(const MachineInstrLoc &Other) const {
+ return BlockNum == Other.BlockNum && Offset == Other.Offset;
+ }
+ };
+
+ MachineInstrLoc CallLocation;
+ std::vector<ArgRegPair> ArgForwardingRegs;
+
+ bool operator==(const CallSiteInfo &Other) const {
+ return CallLocation.BlockNum == Other.CallLocation.BlockNum &&
+ CallLocation.Offset == Other.CallLocation.Offset;
+ }
+};
+
+template <> struct MappingTraits<CallSiteInfo::ArgRegPair> {
+ static void mapping(IO &YamlIO, CallSiteInfo::ArgRegPair &ArgReg) {
+ YamlIO.mapRequired("arg", ArgReg.ArgNo);
+ YamlIO.mapRequired("reg", ArgReg.Reg);
+ }
+
+ static const bool flow = true;
+};
+}
+}
+
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::CallSiteInfo::ArgRegPair)
+
+namespace llvm {
+namespace yaml {
+
+template <> struct MappingTraits<CallSiteInfo> {
+ static void mapping(IO &YamlIO, CallSiteInfo &CSInfo) {
+ YamlIO.mapRequired("bb", CSInfo.CallLocation.BlockNum);
+ YamlIO.mapRequired("offset", CSInfo.CallLocation.Offset);
+ YamlIO.mapOptional("fwdArgRegs", CSInfo.ArgForwardingRegs,
+ std::vector<CallSiteInfo::ArgRegPair>());
+ }
+
+ static const bool flow = true;
+};
+
struct MachineConstantPoolValue {
UnsignedValue ID;
StringValue Value;
@@ -390,6 +461,7 @@
LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::VirtualRegisterDefinition)
LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::MachineStackObject)
LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::FixedMachineStackObject)
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::CallSiteInfo)
LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::MachineConstantPoolValue)
LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::MachineJumpTable::Entry)
@@ -482,6 +554,20 @@
}
};
+/// Targets should override this in a way that mirrors the implementation of
+/// llvm::MachineFunctionInfo.
+struct MachineFunctionInfo {
+ virtual ~MachineFunctionInfo() {}
+ virtual void mappingImpl(IO &YamlIO) {}
+};
+
+template <> struct MappingTraits<std::unique_ptr<MachineFunctionInfo>> {
+ static void mapping(IO &YamlIO, std::unique_ptr<MachineFunctionInfo> &MFI) {
+ if (MFI)
+ MFI->mappingImpl(YamlIO);
+ }
+};
+
struct MachineFunction {
StringRef Name;
unsigned Alignment = 0;
@@ -503,6 +589,8 @@
std::vector<FixedMachineStackObject> FixedStackObjects;
std::vector<MachineStackObject> StackObjects;
std::vector<MachineConstantPoolValue> Constants; /// Constant pool.
+ std::unique_ptr<MachineFunctionInfo> MachineFuncInfo;
+ std::vector<CallSiteInfo> CallSitesInfo;
MachineJumpTable JumpTableInfo;
BlockStringValue Body;
};
@@ -529,8 +617,11 @@
std::vector<FixedMachineStackObject>());
YamlIO.mapOptional("stack", MF.StackObjects,
std::vector<MachineStackObject>());
+ YamlIO.mapOptional("callSites", MF.CallSitesInfo,
+ std::vector<CallSiteInfo>());
YamlIO.mapOptional("constants", MF.Constants,
std::vector<MachineConstantPoolValue>());
+ YamlIO.mapOptional("machineFunctionInfo", MF.MachineFuncInfo);
if (!YamlIO.outputting() || !MF.JumpTableInfo.Entries.empty())
YamlIO.mapOptional("jumpTable", MF.JumpTableInfo, MachineJumpTable());
YamlIO.mapOptional("body", MF.Body, BlockStringValue());
diff --git a/linux-x64/clang/include/llvm/CodeGen/MachineBasicBlock.h b/linux-x64/clang/include/llvm/CodeGen/MachineBasicBlock.h
index 34e18ba..333d0a7 100644
--- a/linux-x64/clang/include/llvm/CodeGen/MachineBasicBlock.h
+++ b/linux-x64/clang/include/llvm/CodeGen/MachineBasicBlock.h
@@ -114,6 +114,10 @@
/// branch.
bool AddressTaken = false;
+ /// Indicate that this basic block needs its symbol be emitted regardless of
+ /// whether the flow just falls-through to it.
+ bool LabelMustBeEmitted = false;
+
/// Indicate that this basic block is the entry block of an EH scope, i.e.,
/// the block that used to have a catchpad or cleanuppad instruction in the
/// LLVM IR.
@@ -158,6 +162,13 @@
/// branch.
void setHasAddressTaken() { AddressTaken = true; }
+ /// Test whether this block must have its label emitted.
+ bool hasLabelMustBeEmitted() const { return LabelMustBeEmitted; }
+
+ /// Set this block to reflect that, regardless how we flow to it, we need
+ /// its label be emitted.
+ void setLabelMustBeEmitted() { LabelMustBeEmitted = true; }
+
/// Return the MachineFunction containing this basic block.
const MachineFunction *getParent() const { return xParent; }
MachineFunction *getParent() { return xParent; }
@@ -899,11 +910,11 @@
MachineBasicBlock::iterator I, B, E;
public:
- MachineInstrSpan(MachineBasicBlock::iterator I)
- : MBB(*I->getParent()),
- I(I),
- B(I == MBB.begin() ? MBB.end() : std::prev(I)),
- E(std::next(I)) {}
+ MachineInstrSpan(MachineBasicBlock::iterator I, MachineBasicBlock *BB)
+ : MBB(*BB), I(I), B(I == MBB.begin() ? MBB.end() : std::prev(I)),
+ E(std::next(I)) {
+ assert(I == BB->end() || I->getParent() == BB);
+ }
MachineBasicBlock::iterator begin() {
return B == MBB.end() ? MBB.begin() : std::next(B);
diff --git a/linux-x64/clang/include/llvm/CodeGen/MachineFrameInfo.h b/linux-x64/clang/include/llvm/CodeGen/MachineFrameInfo.h
index 402a2dc..7617351 100644
--- a/linux-x64/clang/include/llvm/CodeGen/MachineFrameInfo.h
+++ b/linux-x64/clang/include/llvm/CodeGen/MachineFrameInfo.h
@@ -470,7 +470,10 @@
assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() &&
"Invalid Object Idx!");
Objects[ObjectIdx+NumFixedObjects].Alignment = Align;
- ensureMaxAlignment(Align);
+
+ // Only ensure max alignment for the default stack.
+ if (getStackID(ObjectIdx) == 0)
+ ensureMaxAlignment(Align);
}
/// Return the underlying Alloca of the specified
@@ -697,6 +700,8 @@
assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() &&
"Invalid Object Idx!");
Objects[ObjectIdx+NumFixedObjects].StackID = ID;
+ // If ID > 0, MaxAlignment may now be overly conservative.
+ // If ID == 0, MaxAlignment will need to be updated separately.
}
/// Returns true if the specified index corresponds to a dead object.
diff --git a/linux-x64/clang/include/llvm/CodeGen/MachineFunction.h b/linux-x64/clang/include/llvm/CodeGen/MachineFunction.h
index 34ceb15..201c126 100644
--- a/linux-x64/clang/include/llvm/CodeGen/MachineFunction.h
+++ b/linux-x64/clang/include/llvm/CodeGen/MachineFunction.h
@@ -30,11 +30,6 @@
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineMemOperand.h"
-#include "llvm/IR/DebugLoc.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/Metadata.h"
-#include "llvm/MC/MCDwarf.h"
-#include "llvm/MC/MCSymbol.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/ArrayRecycler.h"
#include "llvm/Support/AtomicOrdering.h"
@@ -52,6 +47,7 @@
class BasicBlock;
class BlockAddress;
class DataLayout;
+class DebugLoc;
class DIExpression;
class DILocalVariable;
class DILocation;
@@ -66,6 +62,7 @@
class MachineRegisterInfo;
class MCContext;
class MCInstrDesc;
+class MCSymbol;
class Pass;
class PseudoSourceValueManager;
class raw_ostream;
@@ -324,6 +321,10 @@
/// CodeView label annotations.
std::vector<std::pair<MCSymbol *, MDNode *>> CodeViewAnnotations;
+ /// CodeView heapallocsites.
+ std::vector<std::tuple<MCSymbol*, MCSymbol*, DIType*>>
+ CodeViewHeapAllocSites;
+
bool CallsEHReturn = false;
bool CallsUnwindInit = false;
bool HasEHScopes = false;
@@ -377,9 +378,28 @@
virtual void MF_HandleRemoval(MachineInstr &MI) = 0;
};
+ /// Structure used to represent pair of argument number after call lowering
+ /// and register used to transfer that argument.
+ /// For now we support only cases when argument is transferred through one
+ /// register.
+ struct ArgRegPair {
+ unsigned Reg;
+ uint16_t ArgNo;
+ ArgRegPair(unsigned R, unsigned Arg) : Reg(R), ArgNo(Arg) {
+ assert(Arg < (1 << 16) && "Arg out of range");
+ }
+ };
+ /// Vector of call argument and its forwarding register.
+ using CallSiteInfo = SmallVector<ArgRegPair, 1>;
+ using CallSiteInfoImpl = SmallVectorImpl<ArgRegPair>;
+
private:
Delegate *TheDelegate = nullptr;
+ using CallSiteInfoMap = DenseMap<const MachineInstr *, CallSiteInfo>;
+ /// Map a call instruction to call site arguments forwarding info.
+ CallSiteInfoMap CallSitesInfo;
+
// Callbacks for insertion and removal.
void handleInsertion(MachineInstr &MI);
void handleRemoval(MachineInstr &MI);
@@ -442,7 +462,6 @@
/// getSubtarget - Return the subtarget for which this machine code is being
/// compiled.
const TargetSubtargetInfo &getSubtarget() const { return *STI; }
- void setSubtarget(const TargetSubtargetInfo *ST) { STI = ST; }
/// getSubtarget - This method returns a pointer to the specified type of
/// TargetSubtargetInfo. In debug builds, it verifies that the object being
@@ -740,6 +759,12 @@
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const AAMDNodes &AAInfo);
+ /// Allocate a new MachineMemOperand by copying an existing one,
+ /// replacing the flags. MachineMemOperands are owned
+ /// by the MachineFunction and need not be explicitly deallocated.
+ MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
+ MachineMemOperand::Flags Flags);
+
using OperandCapacity = ArrayRecycler<MachineOperand>::Capacity;
/// Allocate an array of MachineOperands. This is only intended for use by
@@ -790,10 +815,7 @@
return FrameInstructions;
}
- LLVM_NODISCARD unsigned addFrameInst(const MCCFIInstruction &Inst) {
- FrameInstructions.push_back(Inst);
- return FrameInstructions.size() - 1;
- }
+ LLVM_NODISCARD unsigned addFrameInst(const MCCFIInstruction &Inst);
/// \name Exception Handling
/// \{
@@ -912,6 +934,14 @@
return CodeViewAnnotations;
}
+ /// Record heapallocsites
+ void addCodeViewHeapAllocSite(MachineInstr *I, MDNode *MD);
+
+ ArrayRef<std::tuple<MCSymbol*, MCSymbol*, DIType*>>
+ getCodeViewHeapAllocSites() const {
+ return CodeViewHeapAllocSites;
+ }
+
/// Return a reference to the C++ typeinfo for the current function.
const std::vector<const GlobalValue *> &getTypeInfos() const {
return TypeInfos;
@@ -935,6 +965,23 @@
const VariableDbgInfoMapTy &getVariableDbgInfo() const {
return VariableDbgInfos;
}
+
+ void addCallArgsForwardingRegs(const MachineInstr *CallI,
+ CallSiteInfoImpl &&CallInfo) {
+ assert(CallI->isCall());
+ CallSitesInfo[CallI] = std::move(CallInfo);
+ }
+
+ const CallSiteInfoMap &getCallSitesInfo() const {
+ return CallSitesInfo;
+ }
+
+ /// Update call sites info by deleting entry for \p Old call instruction.
+ /// If \p New is present then transfer \p Old call info to it. This function
+ /// should be called before removing call instruction or before replacing
+ /// call instruction with new one.
+ void updateCallSiteInfo(const MachineInstr *Old,
+ const MachineInstr *New = nullptr);
};
//===--------------------------------------------------------------------===//
diff --git a/linux-x64/clang/include/llvm/CodeGen/MachineInstr.h b/linux-x64/clang/include/llvm/CodeGen/MachineInstr.h
index 4cb39c5..c82c5b1 100644
--- a/linux-x64/clang/include/llvm/CodeGen/MachineInstr.h
+++ b/linux-x64/clang/include/llvm/CodeGen/MachineInstr.h
@@ -24,6 +24,7 @@
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/TargetOpcodes.h"
+#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/InlineAsm.h"
#include "llvm/MC/MCInstrDesc.h"
@@ -102,8 +103,10 @@
// no unsigned wrap.
NoSWrap = 1 << 12, // Instruction supports binary operator
// no signed wrap.
- IsExact = 1 << 13 // Instruction supports division is
+ IsExact = 1 << 13, // Instruction supports division is
// known to be exact.
+ FPExcept = 1 << 14, // Instruction may raise floating-point
+ // exceptions.
};
private:
@@ -830,6 +833,17 @@
return mayLoad(Type) || mayStore(Type);
}
+ /// Return true if this instruction could possibly raise a floating-point
+ /// exception. This is the case if the instruction is a floating-point
+ /// instruction that can in principle raise an exception, as indicated
+ /// by the MCID::MayRaiseFPException property, *and* at the same time,
+ /// the instruction is used in a context where we expect floating-point
+ /// exceptions might be enabled, as indicated by the FPExcept MI flag.
+ bool mayRaiseFPException() const {
+ return hasProperty(MCID::MayRaiseFPException) &&
+ getFlag(MachineInstr::MIFlag::FPExcept);
+ }
+
//===--------------------------------------------------------------------===//
// Flags that indicate whether an instruction can be modified by a method.
//===--------------------------------------------------------------------===//
@@ -1005,6 +1019,18 @@
&& getOperand(1).isImm();
}
+ /// A DBG_VALUE is an entry value iff its debug expression contains the
+ /// DW_OP_entry_value DWARF operation.
+ bool isDebugEntryValue() const {
+ return isDebugValue() && getDebugExpression()->isEntryValue();
+ }
+
+ /// Return true if the instruction is a debug value which describes a part of
+ /// a variable as unavailable.
+ bool isUndefDebugValue() const {
+ return isDebugValue() && getOperand(0).isReg() && !getOperand(0).getReg();
+ }
+
bool isPHI() const {
return getOpcode() == TargetOpcode::PHI ||
getOpcode() == TargetOpcode::G_PHI;
@@ -1016,6 +1042,8 @@
getOpcode() == TargetOpcode::INLINEASM_BR;
}
+ /// FIXME: Seems like a layering violation that the AsmDialect, which is X86
+ /// specific, be attached to a generic MachineInstr.
bool isMSInlineAsm() const {
return isInlineAsm() && getInlineAsmDialect() == InlineAsm::AD_Intel;
}
@@ -1199,12 +1227,22 @@
/// Wrapper for findRegisterDefOperandIdx, it returns
/// a pointer to the MachineOperand rather than an index.
- MachineOperand *findRegisterDefOperand(unsigned Reg, bool isDead = false,
- const TargetRegisterInfo *TRI = nullptr) {
- int Idx = findRegisterDefOperandIdx(Reg, isDead, false, TRI);
+ MachineOperand *
+ findRegisterDefOperand(unsigned Reg, bool isDead = false,
+ bool Overlap = false,
+ const TargetRegisterInfo *TRI = nullptr) {
+ int Idx = findRegisterDefOperandIdx(Reg, isDead, Overlap, TRI);
return (Idx == -1) ? nullptr : &getOperand(Idx);
}
+ const MachineOperand *
+ findRegisterDefOperand(unsigned Reg, bool isDead = false,
+ bool Overlap = false,
+ const TargetRegisterInfo *TRI = nullptr) const {
+ return const_cast<MachineInstr *>(this)->findRegisterDefOperand(
+ Reg, isDead, Overlap, TRI);
+ }
+
/// Find the index of the first operand in the
/// operand list that is used to represent the predicate. It returns -1 if
/// none is found.
@@ -1366,7 +1404,7 @@
/// @param AA Optional alias analysis, used to compare memory operands.
/// @param Other MachineInstr to check aliasing against.
/// @param UseTBAA Whether to pass TBAA information to alias analysis.
- bool mayAlias(AliasAnalysis *AA, MachineInstr &Other, bool UseTBAA);
+ bool mayAlias(AliasAnalysis *AA, const MachineInstr &Other, bool UseTBAA) const;
/// Return true if this instruction may have an ordered
/// or volatile memory reference, or if the information describing the memory
@@ -1536,6 +1574,10 @@
/// FIXME: This is not fully implemented yet.
void setPostInstrSymbol(MachineFunction &MF, MCSymbol *Symbol);
+ /// Clone another MachineInstr's pre- and post- instruction symbols and
+ /// replace ours with it.
+ void cloneInstrSymbols(MachineFunction &MF, const MachineInstr &MI);
+
/// Return the MIFlags which represent both MachineInstrs. This
/// should be used when merging two MachineInstrs into one. This routine does
/// not modify the MIFlags of this MachineInstr.
diff --git a/linux-x64/clang/include/llvm/CodeGen/MachineInstrBuilder.h b/linux-x64/clang/include/llvm/CodeGen/MachineInstrBuilder.h
index 4a8cd68..6d7fb72 100644
--- a/linux-x64/clang/include/llvm/CodeGen/MachineInstrBuilder.h
+++ b/linux-x64/clang/include/llvm/CodeGen/MachineInstrBuilder.h
@@ -82,7 +82,7 @@
/// Get the register for the operand index.
/// The operand at the index should be a register (asserted by
/// MachineOperand).
- unsigned getReg(unsigned Idx) { return MI->getOperand(Idx).getReg(); }
+ Register getReg(unsigned Idx) const { return MI->getOperand(Idx).getReg(); }
/// Add a new virtual register operand.
const MachineInstrBuilder &addReg(unsigned RegNo, unsigned flags = 0,
@@ -287,6 +287,9 @@
case MachineOperand::MO_GlobalAddress:
return addGlobalAddress(Disp.getGlobal(), Disp.getOffset() + off,
TargetFlags);
+ case MachineOperand::MO_BlockAddress:
+ return addBlockAddress(Disp.getBlockAddress(), Disp.getOffset() + off,
+ TargetFlags);
}
}
diff --git a/linux-x64/clang/include/llvm/CodeGen/MachineInstrBundle.h b/linux-x64/clang/include/llvm/CodeGen/MachineInstrBundle.h
index 9bb5b63..1810d23 100644
--- a/linux-x64/clang/include/llvm/CodeGen/MachineInstrBundle.h
+++ b/linux-x64/clang/include/llvm/CodeGen/MachineInstrBundle.h
@@ -61,7 +61,8 @@
MachineBasicBlock::instr_iterator I) {
while (I->isBundledWithSucc())
++I;
- return ++I;
+ ++I;
+ return I;
}
/// Returns an iterator pointing beyond the bundle containing \p I.
@@ -69,7 +70,8 @@
MachineBasicBlock::const_instr_iterator I) {
while (I->isBundledWithSucc())
++I;
- return ++I;
+ ++I;
+ return I;
}
//===----------------------------------------------------------------------===//
diff --git a/linux-x64/clang/include/llvm/CodeGen/MachineMemOperand.h b/linux-x64/clang/include/llvm/CodeGen/MachineMemOperand.h
index 12fae2f..65f7063 100644
--- a/linux-x64/clang/include/llvm/CodeGen/MachineMemOperand.h
+++ b/linux-x64/clang/include/llvm/CodeGen/MachineMemOperand.h
@@ -18,8 +18,6 @@
#include "llvm/ADT/BitmaskEnum.h"
#include "llvm/ADT/PointerUnion.h"
#include "llvm/CodeGen/PseudoSourceValue.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/Metadata.h"
#include "llvm/IR/Value.h" // PointerLikeTypeTraits<Value*>
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/DataTypes.h"
@@ -222,6 +220,9 @@
/// Return the size in bytes of the memory reference.
uint64_t getSize() const { return Size; }
+ /// Return the size in bits of the memory reference.
+ uint64_t getSizeInBits() const { return Size * 8; }
+
/// Return the minimum known alignment in bytes of the actual memory
/// reference.
uint64_t getAlignment() const;
diff --git a/linux-x64/clang/include/llvm/CodeGen/MachineModuleInfo.h b/linux-x64/clang/include/llvm/CodeGen/MachineModuleInfo.h
index 9b81dc6..4ff5c7f 100644
--- a/linux-x64/clang/include/llvm/CodeGen/MachineModuleInfo.h
+++ b/linux-x64/clang/include/llvm/CodeGen/MachineModuleInfo.h
@@ -150,6 +150,8 @@
bool doInitialization(Module &) override;
bool doFinalization(Module &) override;
+ const LLVMTargetMachine &getTarget() const { return TM; }
+
const MCContext &getContext() const { return Context; }
MCContext &getContext() { return Context; }
diff --git a/linux-x64/clang/include/llvm/CodeGen/MachineOperand.h b/linux-x64/clang/include/llvm/CodeGen/MachineOperand.h
index ddcdc07..2152c75 100644
--- a/linux-x64/clang/include/llvm/CodeGen/MachineOperand.h
+++ b/linux-x64/clang/include/llvm/CodeGen/MachineOperand.h
@@ -14,6 +14,7 @@
#define LLVM_CODEGEN_MACHINEOPERAND_H
#include "llvm/ADT/DenseMap.h"
+#include "llvm/CodeGen/Register.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/Support/DataTypes.h"
#include "llvm/Support/LowLevelTypeImpl.h"
@@ -345,9 +346,9 @@
//===--------------------------------------------------------------------===//
/// getReg - Returns the register number.
- unsigned getReg() const {
+ Register getReg() const {
assert(isReg() && "This is not a register operand!");
- return SmallContents.RegNo;
+ return Register(SmallContents.RegNo);
}
unsigned getSubReg() const {
@@ -683,6 +684,11 @@
Contents.RegMask = RegMaskPtr;
}
+ void setPredicate(unsigned Predicate) {
+ assert(isPredicate() && "Wrong MachineOperand mutator");
+ Contents.Pred = Predicate;
+ }
+
//===--------------------------------------------------------------------===//
// Other methods.
//===--------------------------------------------------------------------===//
@@ -713,6 +719,10 @@
/// ChangeToES - Replace this operand with a new external symbol operand.
void ChangeToES(const char *SymName, unsigned char TargetFlags = 0);
+ /// ChangeToGA - Replace this operand with a new global address operand.
+ void ChangeToGA(const GlobalValue *GV, int64_t Offset,
+ unsigned char TargetFlags = 0);
+
/// ChangeToMCSymbol - Replace this operand with a new MC symbol operand.
void ChangeToMCSymbol(MCSymbol *Sym);
diff --git a/linux-x64/clang/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h b/linux-x64/clang/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h
index 9a0fd56..a461a29 100644
--- a/linux-x64/clang/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h
+++ b/linux-x64/clang/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h
@@ -158,9 +158,10 @@
/// (1) to filter trivial false positives or (2) to provide more context so
/// that non-trivial false positives can be quickly detected by the user.
bool allowExtraAnalysis(StringRef PassName) const {
- return (MF.getFunction().getContext().getDiagnosticsOutputFile() ||
- MF.getFunction().getContext()
- .getDiagHandlerPtr()->isAnyRemarkEnabled(PassName));
+ return (
+ MF.getFunction().getContext().getRemarkStreamer() ||
+ MF.getFunction().getContext().getDiagHandlerPtr()->isAnyRemarkEnabled(
+ PassName));
}
/// Take a lambda that returns a remark which will be emitted. Second
@@ -171,8 +172,11 @@
// remarks enabled. We can't currently check whether remarks are requested
// for the calling pass since that requires actually building the remark.
- if (MF.getFunction().getContext().getDiagnosticsOutputFile() ||
- MF.getFunction().getContext().getDiagHandlerPtr()->isAnyRemarkEnabled()) {
+ if (MF.getFunction().getContext().getRemarkStreamer() ||
+ MF.getFunction()
+ .getContext()
+ .getDiagHandlerPtr()
+ ->isAnyRemarkEnabled()) {
auto R = RemarkBuilder();
emit((DiagnosticInfoOptimizationBase &)R);
}
diff --git a/linux-x64/clang/include/llvm/CodeGen/MachinePipeliner.h b/linux-x64/clang/include/llvm/CodeGen/MachinePipeliner.h
index a30e4b9..03ca530 100644
--- a/linux-x64/clang/include/llvm/CodeGen/MachinePipeliner.h
+++ b/linux-x64/clang/include/llvm/CodeGen/MachinePipeliner.h
@@ -318,9 +318,9 @@
MBBVectorTy &EpilogBBs);
void splitLifetimes(MachineBasicBlock *KernelBB, MBBVectorTy &EpilogBBs,
SMSchedule &Schedule);
- void addBranches(MBBVectorTy &PrologBBs, MachineBasicBlock *KernelBB,
- MBBVectorTy &EpilogBBs, SMSchedule &Schedule,
- ValueMapTy *VRMap);
+ void addBranches(MachineBasicBlock &PreheaderBB, MBBVectorTy &PrologBBs,
+ MachineBasicBlock *KernelBB, MBBVectorTy &EpilogBBs,
+ SMSchedule &Schedule, ValueMapTy *VRMap);
bool computeDelta(MachineInstr &MI, unsigned &Delta);
void updateMemOperands(MachineInstr &NewMI, MachineInstr &OldMI,
unsigned Num);
@@ -463,6 +463,56 @@
#endif
};
+// 16 was selected based on the number of ProcResource kinds for all
+// existing Subtargets, so that SmallVector don't need to resize too often.
+static const int DefaultProcResSize = 16;
+
+class ResourceManager {
+private:
+ const MCSubtargetInfo *STI;
+ const MCSchedModel &SM;
+ const bool UseDFA;
+ std::unique_ptr<DFAPacketizer> DFAResources;
+ /// Each processor resource is associated with a so-called processor resource
+ /// mask. This vector allows to correlate processor resource IDs with
+ /// processor resource masks. There is exactly one element per each processor
+ /// resource declared by the scheduling model.
+ llvm::SmallVector<uint64_t, DefaultProcResSize> ProcResourceMasks;
+
+ llvm::SmallVector<uint64_t, DefaultProcResSize> ProcResourceCount;
+
+public:
+ ResourceManager(const TargetSubtargetInfo *ST)
+ : STI(ST), SM(ST->getSchedModel()), UseDFA(ST->useDFAforSMS()),
+ ProcResourceMasks(SM.getNumProcResourceKinds(), 0),
+ ProcResourceCount(SM.getNumProcResourceKinds(), 0) {
+ if (UseDFA)
+ DFAResources.reset(ST->getInstrInfo()->CreateTargetScheduleState(*ST));
+ initProcResourceVectors(SM, ProcResourceMasks);
+ }
+
+ void initProcResourceVectors(const MCSchedModel &SM,
+ SmallVectorImpl<uint64_t> &Masks);
+ /// Check if the resources occupied by a MCInstrDesc are available in
+ /// the current state.
+ bool canReserveResources(const MCInstrDesc *MID) const;
+
+ /// Reserve the resources occupied by a MCInstrDesc and change the current
+ /// state to reflect that change.
+ void reserveResources(const MCInstrDesc *MID);
+
+ /// Check if the resources occupied by a machine instruction are available
+ /// in the current state.
+ bool canReserveResources(const MachineInstr &MI) const;
+
+ /// Reserve the resources occupied by a machine instruction and change the
+ /// current state to reflect that change.
+ void reserveResources(const MachineInstr &MI);
+
+ /// Reset the state
+ void clearResources();
+};
+
/// This class represents the scheduled code. The main data structure is a
/// map from scheduled cycle to instructions. During scheduling, the
/// data structure explicitly represents all stages/iterations. When
@@ -501,12 +551,11 @@
/// Virtual register information.
MachineRegisterInfo &MRI;
- std::unique_ptr<DFAPacketizer> Resources;
+ ResourceManager ProcItinResources;
public:
SMSchedule(MachineFunction *mf)
- : ST(mf->getSubtarget()), MRI(mf->getRegInfo()),
- Resources(ST.getInstrInfo()->CreateTargetScheduleState(ST)) {}
+ : ST(mf->getSubtarget()), MRI(mf->getRegInfo()), ProcItinResources(&ST) {}
void reset() {
ScheduledInstrs.clear();
diff --git a/linux-x64/clang/include/llvm/CodeGen/MachineRegisterInfo.h b/linux-x64/clang/include/llvm/CodeGen/MachineRegisterInfo.h
index d25cd98..b5deed1 100644
--- a/linux-x64/clang/include/llvm/CodeGen/MachineRegisterInfo.h
+++ b/linux-x64/clang/include/llvm/CodeGen/MachineRegisterInfo.h
@@ -561,9 +561,14 @@
}
/// hasOneNonDBGUse - Return true if there is exactly one non-Debug
- /// instruction using the specified register.
+ /// use of the specified register.
bool hasOneNonDBGUse(unsigned RegNo) const;
+ /// hasOneNonDBGUse - Return true if there is exactly one non-Debug
+ /// instruction using the specified register. Said instruction may have
+ /// multiple uses.
+ bool hasOneNonDBGUser(unsigned RegNo) const;
+
/// replaceRegWith - Replace all instances of FromReg with ToReg in the
/// machine function. This is like llvm-level X->replaceAllUsesWith(Y),
/// except that it also changes any definitions of the register as well.
@@ -712,12 +717,12 @@
/// createVirtualRegister - Create and return a new virtual register in the
/// function with the specified register class.
- unsigned createVirtualRegister(const TargetRegisterClass *RegClass,
+ Register createVirtualRegister(const TargetRegisterClass *RegClass,
StringRef Name = "");
/// Create and return a new virtual register in the function with the same
/// attributes as the given register.
- unsigned cloneVirtualRegister(unsigned VReg, StringRef Name = "");
+ Register cloneVirtualRegister(Register VReg, StringRef Name = "");
/// Get the low-level type of \p Reg or LLT{} if Reg is not a generic
/// (target independent) virtual register.
@@ -732,7 +737,7 @@
/// Create and return a new generic virtual register with low-level
/// type \p Ty.
- unsigned createGenericVirtualRegister(LLT Ty, StringRef Name = "");
+ Register createGenericVirtualRegister(LLT Ty, StringRef Name = "");
/// Remove all types associated to virtual registers (after instruction
/// selection and constraining of all generic virtual registers).
diff --git a/linux-x64/clang/include/llvm/CodeGen/MachineScheduler.h b/linux-x64/clang/include/llvm/CodeGen/MachineScheduler.h
index 7057998..75a334f 100644
--- a/linux-x64/clang/include/llvm/CodeGen/MachineScheduler.h
+++ b/linux-x64/clang/include/llvm/CodeGen/MachineScheduler.h
@@ -264,10 +264,6 @@
LiveIntervals *LIS;
std::unique_ptr<MachineSchedStrategy> SchedImpl;
- /// Topo - A topological ordering for SUnits which permits fast IsReachable
- /// and similar queries.
- ScheduleDAGTopologicalSort Topo;
-
/// Ordered list of DAG postprocessing steps.
std::vector<std::unique_ptr<ScheduleDAGMutation>> Mutations;
@@ -291,7 +287,7 @@
ScheduleDAGMI(MachineSchedContext *C, std::unique_ptr<MachineSchedStrategy> S,
bool RemoveKillFlags)
: ScheduleDAGInstrs(*C->MF, C->MLI, RemoveKillFlags), AA(C->AA),
- LIS(C->LIS), SchedImpl(std::move(S)), Topo(SUnits, &ExitSU) {}
+ LIS(C->LIS), SchedImpl(std::move(S)) {}
// Provide a vtable anchor
~ScheduleDAGMI() override;
@@ -319,17 +315,6 @@
Mutations.push_back(std::move(Mutation));
}
- /// True if an edge can be added from PredSU to SuccSU without creating
- /// a cycle.
- bool canAddEdge(SUnit *SuccSU, SUnit *PredSU);
-
- /// Add a DAG edge to the given SU with the given predecessor
- /// dependence data.
- ///
- /// \returns true if the edge may be added without creating a cycle OR if an
- /// equivalent edge already existed (false indicates failure).
- bool addEdge(SUnit *SuccSU, const SDep &PredDep);
-
MachineBasicBlock::iterator top() const { return CurrentTop; }
MachineBasicBlock::iterator bottom() const { return CurrentBottom; }
@@ -681,6 +666,10 @@
// scheduled instruction.
SmallVector<unsigned, 16> ReservedCycles;
+ // For each PIdx, stores first index into ReservedCycles that corresponds to
+ // it.
+ SmallVector<unsigned, 16> ReservedCyclesIndex;
+
#ifndef NDEBUG
// Remember the greatest possible stall as an upper bound on the number of
// times we should retry the pending queue because of a hazard.
@@ -755,7 +744,11 @@
/// cycle.
unsigned getLatencyStallCycles(SUnit *SU);
- unsigned getNextResourceCycle(unsigned PIdx, unsigned Cycles);
+ unsigned getNextResourceCycleByInstance(unsigned InstanceIndex,
+ unsigned Cycles);
+
+ std::pair<unsigned, unsigned> getNextResourceCycle(unsigned PIdx,
+ unsigned Cycles);
bool checkHazard(SUnit *SU);
@@ -1014,6 +1007,7 @@
/// Callbacks from ScheduleDAGMI:
/// initPolicy -> initialize(DAG) -> registerRoots -> pickNode ...
class PostGenericScheduler : public GenericSchedulerBase {
+protected:
ScheduleDAGMI *DAG;
SchedBoundary Top;
SmallVector<SUnit*, 8> BotRoots;
diff --git a/linux-x64/clang/include/llvm/CodeGen/Passes.h b/linux-x64/clang/include/llvm/CodeGen/Passes.h
index c59473a..d92ee93 100644
--- a/linux-x64/clang/include/llvm/CodeGen/Passes.h
+++ b/linux-x64/clang/include/llvm/CodeGen/Passes.h
@@ -345,8 +345,9 @@
/// pointer or stack pointer index addressing.
extern char &LocalStackSlotAllocationID;
- /// ExpandISelPseudos - This pass expands pseudo-instructions.
- extern char &ExpandISelPseudosID;
+ /// This pass expands pseudo-instructions, reserves registers and adjusts
+ /// machine frame information.
+ extern char &FinalizeISelID;
/// UnpackMachineBundles - This pass unpack machine instruction bundles.
extern char &UnpackMachineBundlesID;
@@ -446,6 +447,9 @@
/// Creates CFI Instruction Inserter pass. \see CFIInstrInserter.cpp
FunctionPass *createCFIInstrInserter();
+ /// Create Hardware Loop pass. \see HardwareLoops.cpp
+ FunctionPass *createHardwareLoopsPass();
+
} // End llvm namespace
#endif
diff --git a/linux-x64/clang/include/llvm/CodeGen/PseudoSourceValue.h b/linux-x64/clang/include/llvm/CodeGen/PseudoSourceValue.h
index 4920f23..4b3cc91 100644
--- a/linux-x64/clang/include/llvm/CodeGen/PseudoSourceValue.h
+++ b/linux-x64/clang/include/llvm/CodeGen/PseudoSourceValue.h
@@ -15,7 +15,6 @@
#include "llvm/ADT/StringMap.h"
#include "llvm/IR/GlobalValue.h"
-#include "llvm/IR/Value.h"
#include "llvm/IR/ValueMap.h"
#include <map>
@@ -123,7 +122,7 @@
bool mayAlias(const MachineFrameInfo *) const override;
};
-/// A specialized pseudo soruce value for holding GlobalValue values.
+/// A specialized pseudo source value for holding GlobalValue values.
class GlobalValuePseudoSourceValue : public CallEntryPseudoSourceValue {
const GlobalValue *GV;
diff --git a/linux-x64/clang/include/llvm/CodeGen/RegAllocRegistry.h b/linux-x64/clang/include/llvm/CodeGen/RegAllocRegistry.h
index 7077aa3..9a63674 100644
--- a/linux-x64/clang/include/llvm/CodeGen/RegAllocRegistry.h
+++ b/linux-x64/clang/include/llvm/CodeGen/RegAllocRegistry.h
@@ -22,29 +22,30 @@
//===----------------------------------------------------------------------===//
///
-/// RegisterRegAlloc class - Track the registration of register allocators.
+/// RegisterRegAllocBase class - Track the registration of register allocators.
///
//===----------------------------------------------------------------------===//
-class RegisterRegAlloc : public MachinePassRegistryNode<FunctionPass *(*)()> {
+template <class SubClass>
+class RegisterRegAllocBase : public MachinePassRegistryNode<FunctionPass *(*)()> {
public:
using FunctionPassCtor = FunctionPass *(*)();
static MachinePassRegistry<FunctionPassCtor> Registry;
- RegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
+ RegisterRegAllocBase(const char *N, const char *D, FunctionPassCtor C)
: MachinePassRegistryNode(N, D, C) {
Registry.Add(this);
}
- ~RegisterRegAlloc() { Registry.Remove(this); }
+ ~RegisterRegAllocBase() { Registry.Remove(this); }
// Accessors.
- RegisterRegAlloc *getNext() const {
- return (RegisterRegAlloc *)MachinePassRegistryNode::getNext();
+ SubClass *getNext() const {
+ return static_cast<SubClass *>(MachinePassRegistryNode::getNext());
}
- static RegisterRegAlloc *getList() {
- return (RegisterRegAlloc *)Registry.getList();
+ static SubClass *getList() {
+ return static_cast<SubClass *>(Registry.getList());
}
static FunctionPassCtor getDefault() { return Registry.getDefault(); }
@@ -56,6 +57,17 @@
}
};
+class RegisterRegAlloc : public RegisterRegAllocBase<RegisterRegAlloc> {
+public:
+ RegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
+ : RegisterRegAllocBase(N, D, C) {}
+};
+
+/// RegisterRegAlloc's global Registry tracks allocator registration.
+template <class T>
+MachinePassRegistry<RegisterRegAlloc::FunctionPassCtor>
+RegisterRegAllocBase<T>::Registry;
+
} // end namespace llvm
#endif // LLVM_CODEGEN_REGALLOCREGISTRY_H
diff --git a/linux-x64/clang/include/llvm/CodeGen/Register.h b/linux-x64/clang/include/llvm/CodeGen/Register.h
new file mode 100644
index 0000000..907c1a9
--- /dev/null
+++ b/linux-x64/clang/include/llvm/CodeGen/Register.h
@@ -0,0 +1,60 @@
+//===-- llvm/CodeGen/Register.h ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_REGISTER_H
+#define LLVM_CODEGEN_REGISTER_H
+
+#include <cassert>
+
+namespace llvm {
+
+/// Wrapper class representing virtual and physical registers. Should be passed
+/// by value.
+class Register {
+ unsigned Reg;
+
+public:
+ Register(unsigned Val = 0): Reg(Val) {}
+
+ /// Return true if the specified register number is in the virtual register
+ /// namespace.
+ bool isVirtual() const {
+ return int(Reg) < 0;
+ }
+
+ /// Return true if the specified register number is in the physical register
+ /// namespace.
+ bool isPhysical() const {
+ return int(Reg) > 0;
+ }
+
+ /// Convert a virtual register number to a 0-based index. The first virtual
+ /// register in a function will get the index 0.
+ unsigned virtRegIndex() const {
+ assert(isVirtual() && "Not a virtual register");
+ return Reg & ~(1u << 31);
+ }
+
+ /// Convert a 0-based index to a virtual register number.
+ /// This is the inverse operation of VirtReg2IndexFunctor below.
+ static Register index2VirtReg(unsigned Index) {
+ return Register(Index | (1u << 31));
+ }
+
+ operator unsigned() const {
+ return Reg;
+ }
+
+ bool isValid() const {
+ return Reg != 0;
+ }
+};
+
+}
+
+#endif
diff --git a/linux-x64/clang/include/llvm/CodeGen/RegisterScavenging.h b/linux-x64/clang/include/llvm/CodeGen/RegisterScavenging.h
index 478ea38..9c48df8 100644
--- a/linux-x64/clang/include/llvm/CodeGen/RegisterScavenging.h
+++ b/linux-x64/clang/include/llvm/CodeGen/RegisterScavenging.h
@@ -157,10 +157,15 @@
/// Returns the scavenged register.
/// This is deprecated as it depends on the quality of the kill flags being
/// present; Use scavengeRegisterBackwards() instead!
+ ///
+ /// If \p AllowSpill is false, fail if a spill is required to make the
+ /// register available, and return NoRegister.
unsigned scavengeRegister(const TargetRegisterClass *RC,
- MachineBasicBlock::iterator I, int SPAdj);
- unsigned scavengeRegister(const TargetRegisterClass *RegClass, int SPAdj) {
- return scavengeRegister(RegClass, MBBI, SPAdj);
+ MachineBasicBlock::iterator I, int SPAdj,
+ bool AllowSpill = true);
+ unsigned scavengeRegister(const TargetRegisterClass *RegClass, int SPAdj,
+ bool AllowSpill = true) {
+ return scavengeRegister(RegClass, MBBI, SPAdj, AllowSpill);
}
/// Make a register of the specific register class available from the current
@@ -169,9 +174,13 @@
/// SPAdj is the stack adjustment due to call frame, it's passed along to
/// eliminateFrameIndex().
/// Returns the scavenged register.
+ ///
+ /// If \p AllowSpill is false, fail if a spill is required to make the
+ /// register available, and return NoRegister.
unsigned scavengeRegisterBackwards(const TargetRegisterClass &RC,
MachineBasicBlock::iterator To,
- bool RestoreAfter, int SPAdj);
+ bool RestoreAfter, int SPAdj,
+ bool AllowSpill = true);
/// Tell the scavenger a register is used.
void setRegUsed(unsigned Reg, LaneBitmask LaneMask = LaneBitmask::getAll());
diff --git a/linux-x64/clang/include/llvm/CodeGen/ScheduleDAG.h b/linux-x64/clang/include/llvm/CodeGen/ScheduleDAG.h
index 68614dd..e004f3b 100644
--- a/linux-x64/clang/include/llvm/CodeGen/ScheduleDAG.h
+++ b/linux-x64/clang/include/llvm/CodeGen/ScheduleDAG.h
@@ -414,7 +414,7 @@
/// dirty.
void setDepthToAtLeast(unsigned NewDepth);
- /// If NewDepth is greater than this node's depth value, set it to be
+ /// If NewHeight is greater than this node's height value, set it to be
/// the new height value. This also recursively marks predecessor nodes
/// dirty.
void setHeightToAtLeast(unsigned NewHeight);
@@ -691,6 +691,12 @@
std::vector<SUnit> &SUnits;
SUnit *ExitSU;
+ // Have any new nodes been added?
+ bool Dirty = false;
+
+ // Outstanding added edges, that have not been applied to the ordering.
+ SmallVector<std::pair<SUnit *, SUnit *>, 16> Updates;
+
/// Maps topological index to the node number.
std::vector<int> Index2Node;
/// Maps the node number to its topological index.
@@ -710,6 +716,11 @@
/// Assigns the topological index to the node n.
void Allocate(int n, int index);
+ /// Fix the ordering, by either recomputing from scratch or by applying
+ /// any outstanding updates. Uses a heuristic to estimate what will be
+ /// cheaper.
+ void FixOrder();
+
public:
ScheduleDAGTopologicalSort(std::vector<SUnit> &SUnits, SUnit *ExitSU);
@@ -734,11 +745,19 @@
/// added from SUnit \p X to SUnit \p Y.
void AddPred(SUnit *Y, SUnit *X);
+ /// Queues an update to the topological ordering to accommodate an edge to
+ /// be added from SUnit \p X to SUnit \p Y.
+ void AddPredQueued(SUnit *Y, SUnit *X);
+
/// Updates the topological ordering to accommodate an an edge to be
/// removed from the specified node \p N from the predecessors of the
/// current node \p M.
void RemovePred(SUnit *M, SUnit *N);
+ /// Mark the ordering as temporarily broken, after a new node has been
+ /// added.
+ void MarkDirty() { Dirty = true; }
+
typedef std::vector<int>::iterator iterator;
typedef std::vector<int>::const_iterator const_iterator;
iterator begin() { return Index2Node.begin(); }
diff --git a/linux-x64/clang/include/llvm/CodeGen/ScheduleDAGInstrs.h b/linux-x64/clang/include/llvm/CodeGen/ScheduleDAGInstrs.h
index fa7f886..3e3b604 100644
--- a/linux-x64/clang/include/llvm/CodeGen/ScheduleDAGInstrs.h
+++ b/linux-x64/clang/include/llvm/CodeGen/ScheduleDAGInstrs.h
@@ -234,6 +234,11 @@
/// For an unanalyzable memory access, this Value is used in maps.
UndefValue *UnknownValue;
+
+ /// Topo - A topological ordering for SUnits which permits fast IsReachable
+ /// and similar queries.
+ ScheduleDAGTopologicalSort Topo;
+
using DbgValueVector =
std::vector<std::pair<MachineInstr *, MachineInstr *>>;
/// Remember instruction that precedes DBG_VALUE.
@@ -338,6 +343,17 @@
/// Fixes register kill flags that scheduling has made invalid.
void fixupKills(MachineBasicBlock &MBB);
+ /// True if an edge can be added from PredSU to SuccSU without creating
+ /// a cycle.
+ bool canAddEdge(SUnit *SuccSU, SUnit *PredSU);
+
+ /// Add a DAG edge to the given SU with the given predecessor
+ /// dependence data.
+ ///
+ /// \returns true if the edge may be added without creating a cycle OR if an
+ /// equivalent edge already existed (false indicates failure).
+ bool addEdge(SUnit *SuccSU, const SDep &PredDep);
+
protected:
void initSUnits();
void addPhysRegDataDeps(SUnit *SU, unsigned OperIdx);
diff --git a/linux-x64/clang/include/llvm/CodeGen/SelectionDAG.h b/linux-x64/clang/include/llvm/CodeGen/SelectionDAG.h
index dfd1d49..12a9708 100644
--- a/linux-x64/clang/include/llvm/CodeGen/SelectionDAG.h
+++ b/linux-x64/clang/include/llvm/CodeGen/SelectionDAG.h
@@ -267,6 +267,10 @@
/// Tracks dbg_value and dbg_label information through SDISel.
SDDbgInfo *DbgInfo;
+ using CallSiteInfo = MachineFunction::CallSiteInfo;
+ using CallSiteInfoImpl = MachineFunction::CallSiteInfoImpl;
+ DenseMap<const SDNode *, CallSiteInfo> SDCallSiteInfo;
+
uint16_t NextPersistentId = 0;
public:
@@ -297,6 +301,9 @@
/// The node N that was updated.
virtual void NodeUpdated(SDNode *N);
+
+ /// The node N that was inserted.
+ virtual void NodeInserted(SDNode *N);
};
struct DAGNodeDeletedListener : public DAGUpdateListener {
@@ -403,6 +410,7 @@
const TargetLowering &getTargetLoweringInfo() const { return *TLI; }
const TargetLibraryInfo &getLibInfo() const { return *LibInfo; }
const SelectionDAGTargetInfo &getSelectionDAGInfo() const { return *TSI; }
+ const LegacyDivergenceAnalysis *getDivergenceAnalysis() const { return DA; }
LLVMContext *getContext() const {return Context; }
OptimizationRemarkEmitter &getORE() const { return *ORE; }
@@ -572,6 +580,9 @@
bool isTarget = false, bool isOpaque = false);
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL,
bool isTarget = false);
+ SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL,
+ bool LegalTypes = true);
+
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT,
bool isOpaque = false) {
return getConstant(Val, DL, VT, true, isOpaque);
@@ -788,6 +799,16 @@
/// value assuming it was the smaller SrcTy value.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT);
+ /// Convert Op, which must be of integer type, to the integer type VT, by
+ /// either truncating it or performing either zero or sign extension as
+ /// appropriate extension for the pointer's semantics.
+ SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
+
+ /// Return the expression required to extend the Op as a pointer value
+ /// assuming it was the smaller SrcTy value. This may be either a zero extend
+ /// or a sign extend.
+ SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT);
+
/// Convert Op, which must be of integer type, to the integer type VT,
/// by using an extension appropriate for the target's
/// BooleanContent for type OpVT or truncating it.
@@ -970,6 +991,10 @@
/// Try to simplify a shift into 1 of its operands or a constant.
SDValue simplifyShift(SDValue X, SDValue Y);
+ /// Try to simplify a floating-point binary operation into 1 of its operands
+ /// or a constant.
+ SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y);
+
/// VAArg produces a result and token chain, and takes a pointer
/// and a source value as input.
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
@@ -981,12 +1006,6 @@
/// a success flag (initially i1), and a chain.
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT,
SDVTList VTs, SDValue Chain, SDValue Ptr,
- SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo,
- unsigned Alignment, AtomicOrdering SuccessOrdering,
- AtomicOrdering FailureOrdering,
- SyncScope::ID SSID);
- SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT,
- SDVTList VTs, SDValue Chain, SDValue Ptr,
SDValue Cmp, SDValue Swp, MachineMemOperand *MMO);
/// Gets a node for an atomic op, produces result (if relevant)
@@ -1016,12 +1035,19 @@
unsigned Align = 0,
MachineMemOperand::Flags Flags
= MachineMemOperand::MOLoad | MachineMemOperand::MOStore,
- unsigned Size = 0);
+ unsigned Size = 0,
+ const AAMDNodes &AAInfo = AAMDNodes());
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList,
ArrayRef<SDValue> Ops, EVT MemVT,
MachineMemOperand *MMO);
+ /// Creates a LifetimeSDNode that starts (`IsStart==true`) or ends
+ /// (`IsStart==false`) the lifetime of the portion of `FrameIndex` between
+ /// offsets `Offset` and `Offset + Size`.
+ SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain,
+ int FrameIndex, int64_t Size, int64_t Offset = -1);
+
/// Create a MERGE_VALUES node from the given operands.
SDValue getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl);
@@ -1404,17 +1430,32 @@
ArrayRef<SDValue> Ops,
const SDNodeFlags Flags = SDNodeFlags());
+ /// Fold floating-point operations with 2 operands when both operands are
+ /// constants and/or undefined.
+ SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT,
+ SDValue N1, SDValue N2);
+
/// Constant fold a setcc to true or false.
SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond,
const SDLoc &dl);
- /// See if the specified operand can be simplified with the knowledge that only
- /// the bits specified by Mask are used. If so, return the simpler operand,
- /// otherwise return a null SDValue.
+ /// See if the specified operand can be simplified with the knowledge that
+ /// only the bits specified by DemandedBits are used. If so, return the
+ /// simpler operand, otherwise return a null SDValue.
///
/// (This exists alongside SimplifyDemandedBits because GetDemandedBits can
/// simplify nodes with multiple uses more aggressively.)
- SDValue GetDemandedBits(SDValue V, const APInt &Mask);
+ SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits);
+
+ /// See if the specified operand can be simplified with the knowledge that
+ /// only the bits specified by DemandedBits are used in the elements specified
+ /// by DemandedElts. If so, return the simpler operand, otherwise return a
+ /// null SDValue.
+ ///
+ /// (This exists alongside SimplifyDemandedBits because GetDemandedBits can
+ /// simplify nodes with multiple uses more aggressively.)
+ SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits,
+ const APInt &DemandedElts);
/// Return true if the sign bit of Op is known to be zero.
/// We use this predicate to simplify operations downstream.
@@ -1423,8 +1464,19 @@
/// Return true if 'Op & Mask' is known to be zero. We
/// use this predicate to simplify operations downstream. Op and Mask are
/// known to be the same type.
- bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth = 0)
- const;
+ bool MaskedValueIsZero(SDValue Op, const APInt &Mask,
+ unsigned Depth = 0) const;
+
+ /// Return true if 'Op & Mask' is known to be zero in DemandedElts. We
+ /// use this predicate to simplify operations downstream. Op and Mask are
+ /// known to be the same type.
+ bool MaskedValueIsZero(SDValue Op, const APInt &Mask,
+ const APInt &DemandedElts, unsigned Depth = 0) const;
+
+ /// Return true if '(Op & Mask) == Mask'.
+ /// Op and Mask are known to be the same type.
+ bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask,
+ unsigned Depth = 0) const;
/// Determine which bits of Op are known to be either zero or one and return
/// them in Known. For vectors, the known bits are those that are shared by
@@ -1524,6 +1576,13 @@
/// Test whether \p V has a splatted value.
bool isSplatValue(SDValue V, bool AllowUndefs = false);
+ /// If V is a splatted value, return the source vector and its splat index.
+ SDValue getSplatSourceVector(SDValue V, int &SplatIndex);
+
+ /// If V is a splat vector, return its scalar source operand by extracting
+ /// that element from the source vector.
+ SDValue getSplatValue(SDValue V);
+
/// Match a binop + shuffle pyramid that represents a horizontal reduction
/// over the elements of a vector starting from the EXTRACT_VECTOR_ELT node /p
/// Extract. The reduction must use one of the opcodes listed in /p
@@ -1541,6 +1600,11 @@
/// vector op and fill the end of the resulting vector with UNDEFS.
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE = 0);
+ /// Like UnrollVectorOp(), but for the [US](ADD|SUB|MUL)O family of opcodes.
+ /// This is a separate function because those opcodes have two results.
+ std::pair<SDValue, SDValue> UnrollVectorOverflowOp(SDNode *N,
+ unsigned ResNE = 0);
+
/// Return true if loads are next to each other and can be
/// merged. Check that both are nonvolatile and if LD is loading
/// 'Bytes' bytes from a location that is 'Dist' units away from the
@@ -1575,6 +1639,9 @@
return SplitVector(N->getOperand(OpNo), SDLoc(N));
}
+ /// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
+ SDValue WidenVector(const SDValue &N, const SDLoc &DL);
+
/// Append the extracted elements from Start to Count out of the vector Op
/// in Args. If Count is 0, all of the elements will be extracted.
void ExtractVectorElements(SDValue Op, SmallVectorImpl<SDValue> &Args,
@@ -1596,6 +1663,17 @@
isConstantFPBuildVectorOrConstantFP(N);
}
+ void addCallSiteInfo(const SDNode *CallNode, CallSiteInfoImpl &&CallInfo) {
+ SDCallSiteInfo[CallNode] = std::move(CallInfo);
+ }
+
+ CallSiteInfo getSDCallSiteInfo(const SDNode *CallNode) {
+ auto I = SDCallSiteInfo.find(CallNode);
+ if (I != SDCallSiteInfo.end())
+ return std::move(I->second);
+ return CallSiteInfo();
+ }
+
private:
void InsertNode(SDNode *N);
bool RemoveNodeFromCSEMaps(SDNode *N);
diff --git a/linux-x64/clang/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h b/linux-x64/clang/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h
index f168b84..4ee5833 100644
--- a/linux-x64/clang/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h
+++ b/linux-x64/clang/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h
@@ -33,11 +33,13 @@
private:
SDValue Base;
SDValue Index;
- int64_t Offset = 0;
+ Optional<int64_t> Offset;
bool IsIndexSignExt = false;
public:
BaseIndexOffset() = default;
+ BaseIndexOffset(SDValue Base, SDValue Index, bool IsIndexSignExt)
+ : Base(Base), Index(Index), Offset(), IsIndexSignExt(IsIndexSignExt) {}
BaseIndexOffset(SDValue Base, SDValue Index, int64_t Offset,
bool IsIndexSignExt)
: Base(Base), Index(Index), Offset(Offset),
@@ -47,6 +49,13 @@
SDValue getBase() const { return Base; }
SDValue getIndex() { return Index; }
SDValue getIndex() const { return Index; }
+ bool hasValidOffset() const { return Offset.hasValue(); }
+
+ // Returns true if `Other` and `*this` are both some offset from the same base
+ // pointer. In that case, `Off` is set to the offset between `*this` and
+ // `Other` (negative if `Other` is before `*this`).
+ bool equalBaseIndex(const BaseIndexOffset &Other, const SelectionDAG &DAG,
+ int64_t &Off) const;
bool equalBaseIndex(const BaseIndexOffset &Other,
const SelectionDAG &DAG) const {
@@ -54,11 +63,28 @@
return equalBaseIndex(Other, DAG, Off);
}
- bool equalBaseIndex(const BaseIndexOffset &Other, const SelectionDAG &DAG,
- int64_t &Off) const;
+ // Returns true if `Other` (with size `OtherSize`) can be proven to be fully
+ // contained in `*this` (with size `Size`).
+ bool contains(const SelectionDAG &DAG, int64_t BitSize,
+ const BaseIndexOffset &Other, int64_t OtherBitSize,
+ int64_t &BitOffset) const;
- /// Parses tree in Ptr for base, index, offset addresses.
- static BaseIndexOffset match(const LSBaseSDNode *N, const SelectionDAG &DAG);
+ bool contains(const SelectionDAG &DAG, int64_t BitSize,
+ const BaseIndexOffset &Other, int64_t OtherBitSize) const {
+ int64_t BitOffset;
+ return contains(DAG, BitSize, Other, OtherBitSize, BitOffset);
+ }
+
+ // Returns true `Op0` and `Op1` can be proven to alias/not alias, in
+ // which case `IsAlias` is set to true/false.
+ static bool computeAliasing(const SDNode *Op0,
+ const Optional<int64_t> NumBytes0,
+ const SDNode *Op1,
+ const Optional<int64_t> NumBytes1,
+ const SelectionDAG &DAG, bool &IsAlias);
+
+ /// Parses tree in N for base, index, offset addresses.
+ static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG);
void print(raw_ostream& OS) const;
void dump() const;
diff --git a/linux-x64/clang/include/llvm/CodeGen/SelectionDAGISel.h b/linux-x64/clang/include/llvm/CodeGen/SelectionDAGISel.h
index 2acb922..147c325 100644
--- a/linux-x64/clang/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/linux-x64/clang/include/llvm/CodeGen/SelectionDAGISel.h
@@ -34,6 +34,7 @@
class TargetLibraryInfo;
class FunctionLoweringInfo;
class ScheduleHazardRecognizer;
+ class SwiftErrorValueTracking;
class GCFunctionInfo;
class ScheduleDAGSDNodes;
class LoadInst;
@@ -45,6 +46,7 @@
TargetMachine &TM;
const TargetLibraryInfo *LibInfo;
FunctionLoweringInfo *FuncInfo;
+ SwiftErrorValueTracking *SwiftError;
MachineFunction *MF;
MachineRegisterInfo *RegInfo;
SelectionDAG *CurDAG;
@@ -143,10 +145,12 @@
OPC_CheckInteger,
OPC_CheckChild0Integer, OPC_CheckChild1Integer, OPC_CheckChild2Integer,
OPC_CheckChild3Integer, OPC_CheckChild4Integer,
- OPC_CheckCondCode,
+ OPC_CheckCondCode, OPC_CheckChild2CondCode,
OPC_CheckValueType,
OPC_CheckComplexPat,
OPC_CheckAndImm, OPC_CheckOrImm,
+ OPC_CheckImmAllOnesV,
+ OPC_CheckImmAllZerosV,
OPC_CheckFoldableChainNode,
OPC_EmitInteger,
diff --git a/linux-x64/clang/include/llvm/CodeGen/SelectionDAGNodes.h b/linux-x64/clang/include/llvm/CodeGen/SelectionDAGNodes.h
index c0dd9d1..5aab964 100644
--- a/linux-x64/clang/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/linux-x64/clang/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -368,6 +368,13 @@
bool ApproximateFuncs : 1;
bool AllowReassociation : 1;
+ // We assume instructions do not raise floating-point exceptions by default,
+ // and only those marked explicitly may do so. We could choose to represent
+ // this via a positive "FPExcept" flags like on the MI level, but having a
+ // negative "NoFPExcept" flag here (that defaults to true) makes the flag
+ // intersection logic more straightforward.
+ bool NoFPExcept : 1;
+
public:
/// Default constructor turns off all optimization flags.
SDNodeFlags()
@@ -375,7 +382,7 @@
Exact(false), NoNaNs(false), NoInfs(false),
NoSignedZeros(false), AllowReciprocal(false), VectorReduction(false),
AllowContract(false), ApproximateFuncs(false),
- AllowReassociation(false) {}
+ AllowReassociation(false), NoFPExcept(true) {}
/// Propagate the fast-math-flags from an IR FPMathOperator.
void copyFMF(const FPMathOperator &FPMO) {
@@ -438,6 +445,10 @@
setDefined();
AllowReassociation = b;
}
+ void setFPExcept(bool b) {
+ setDefined();
+ NoFPExcept = !b;
+ }
// These are accessors for each flag.
bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
@@ -451,9 +462,10 @@
bool hasAllowContract() const { return AllowContract; }
bool hasApproximateFuncs() const { return ApproximateFuncs; }
bool hasAllowReassociation() const { return AllowReassociation; }
+ bool hasFPExcept() const { return !NoFPExcept; }
bool isFast() const {
- return NoSignedZeros && AllowReciprocal && NoNaNs && NoInfs &&
+ return NoSignedZeros && AllowReciprocal && NoNaNs && NoInfs && NoFPExcept &&
AllowContract && ApproximateFuncs && AllowReassociation;
}
@@ -473,6 +485,7 @@
AllowContract &= Flags.AllowContract;
ApproximateFuncs &= Flags.ApproximateFuncs;
AllowReassociation &= Flags.AllowReassociation;
+ NoFPExcept &= Flags.NoFPExcept;
}
};
@@ -488,6 +501,17 @@
// SubclassData. These are designed to fit within a uint16_t so they pack
// with NodeType.
+#if defined(_AIX) && (!defined(__GNUC__) || defined(__ibmxl__))
+// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
+// and give the `pack` pragma push semantics.
+#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")
+#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")
+#else
+#define BEGIN_TWO_BYTE_PACK()
+#define END_TWO_BYTE_PACK()
+#endif
+
+BEGIN_TWO_BYTE_PACK()
class SDNodeBitfields {
friend class SDNode;
friend class MemIntrinsicSDNode;
@@ -560,6 +584,9 @@
LoadSDNodeBitfields LoadSDNodeBits;
StoreSDNodeBitfields StoreSDNodeBits;
};
+END_TWO_BYTE_PACK()
+#undef BEGIN_TWO_BYTE_PACK
+#undef END_TWO_BYTE_PACK
// RawSDNodeBits must cover the entirety of the union. This means that all of
// the union's members must have size <= RawSDNodeBits. We write the RHS as
@@ -677,6 +704,8 @@
case ISD::STRICT_FFLOOR:
case ISD::STRICT_FROUND:
case ISD::STRICT_FTRUNC:
+ case ISD::STRICT_FP_ROUND:
+ case ISD::STRICT_FP_EXTEND:
return true;
}
}
@@ -1385,7 +1414,10 @@
public:
AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
EVT MemVT, MachineMemOperand *MMO)
- : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {}
+ : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
+ assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||
+ MMO->isAtomic()) && "then why are we using an AtomicSDNode?");
+ }
const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getVal() const { return getOperand(2); }
@@ -1482,14 +1514,16 @@
bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }
- int getSplatIndex() const {
+ int getSplatIndex() const {
assert(isSplat() && "Cannot get splat index for non-splat!");
EVT VT = getValueType(0);
- for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
+ for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
if (Mask[i] >= 0)
return Mask[i];
- }
- llvm_unreachable("Splat with all undef indices?");
+
+ // We can choose any index value here and be correct because all elements
+ // are undefined. Return 0 for better potential for callers to simplify.
+ return 0;
}
static bool isSplatMask(const int *Mask, EVT VT);
@@ -1622,16 +1656,32 @@
/// If \p V is not a bitcasted one-use value, it is returned as-is.
SDValue peekThroughOneUseBitcasts(SDValue V);
+/// Return the non-extracted vector source operand of \p V if it exists.
+/// If \p V is not an extracted subvector, it is returned as-is.
+SDValue peekThroughExtractSubvectors(SDValue V);
+
/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
/// constant is canonicalized to be operand 1.
-bool isBitwiseNot(SDValue V);
+bool isBitwiseNot(SDValue V, bool AllowUndefs = false);
/// Returns the SDNode if it is a constant splat BuildVector or constant int.
-ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false);
+ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
+ bool AllowTruncation = false);
+
+/// Returns the SDNode if it is a demanded constant splat BuildVector or
+/// constant int.
+ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
+ bool AllowUndefs = false,
+ bool AllowTruncation = false);
/// Returns the SDNode if it is a constant splat BuildVector or constant float.
ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);
+/// Returns the SDNode if it is a demanded constant splat BuildVector or
+/// constant float.
+ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
+ bool AllowUndefs = false);
+
/// Return true if the value is a constant 0 integer or a splatted vector of
/// a constant 0 integer (with no undefs by default).
/// Build vector implicit truncation is not an issue for null values.
@@ -1692,6 +1742,38 @@
}
};
+/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate
+/// the offet and size that are started/ended in the underlying FrameIndex.
+class LifetimeSDNode : public SDNode {
+ friend class SelectionDAG;
+ int64_t Size;
+ int64_t Offset; // -1 if offset is unknown.
+
+ LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
+ SDVTList VTs, int64_t Size, int64_t Offset)
+ : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {}
+public:
+ int64_t getFrameIndex() const {
+ return cast<FrameIndexSDNode>(getOperand(1))->getIndex();
+ }
+
+ bool hasOffset() const { return Offset >= 0; }
+ int64_t getOffset() const {
+ assert(hasOffset() && "offset is unknown");
+ return Offset;
+ }
+ int64_t getSize() const {
+ assert(hasOffset() && "offset is unknown");
+ return Size;
+ }
+
+ // Methods to support isa and dyn_cast
+ static bool classof(const SDNode *N) {
+ return N->getOpcode() == ISD::LIFETIME_START ||
+ N->getOpcode() == ISD::LIFETIME_END;
+ }
+};
+
class JumpTableSDNode : public SDNode {
friend class SelectionDAG;
@@ -1837,12 +1919,31 @@
unsigned MinSplatBits = 0,
bool isBigEndian = false) const;
+ /// Returns the demanded splatted value or a null value if this is not a
+ /// splat.
+ ///
+ /// The DemandedElts mask indicates the elements that must be in the splat.
+ /// If passed a non-null UndefElements bitvector, it will resize it to match
+ /// the vector width and set the bits where elements are undef.
+ SDValue getSplatValue(const APInt &DemandedElts,
+ BitVector *UndefElements = nullptr) const;
+
/// Returns the splatted value or a null value if this is not a splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
SDValue getSplatValue(BitVector *UndefElements = nullptr) const;
+ /// Returns the demanded splatted constant or null if this is not a constant
+ /// splat.
+ ///
+ /// The DemandedElts mask indicates the elements that must be in the splat.
+ /// If passed a non-null UndefElements bitvector, it will resize it to match
+ /// the vector width and set the bits where elements are undef.
+ ConstantSDNode *
+ getConstantSplatNode(const APInt &DemandedElts,
+ BitVector *UndefElements = nullptr) const;
+
/// Returns the splatted constant or null if this is not a constant
/// splat.
///
@@ -1851,6 +1952,16 @@
ConstantSDNode *
getConstantSplatNode(BitVector *UndefElements = nullptr) const;
+ /// Returns the demanded splatted constant FP or null if this is not a
+ /// constant FP splat.
+ ///
+ /// The DemandedElts mask indicates the elements that must be in the splat.
+ /// If passed a non-null UndefElements bitvector, it will resize it to match
+ /// the vector width and set the bits where elements are undef.
+ ConstantFPSDNode *
+ getConstantFPSplatNode(const APInt &DemandedElts,
+ BitVector *UndefElements = nullptr) const;
+
/// Returns the splatted constant FP or null if this is not a constant
/// FP splat.
///
@@ -1975,8 +2086,10 @@
MCSymbol *Label;
- LabelSDNode(unsigned Order, const DebugLoc &dl, MCSymbol *L)
- : SDNode(ISD::EH_LABEL, Order, dl, getSDVTList(MVT::Other)), Label(L) {}
+ LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L)
+ : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) {
+ assert(LabelSDNode::classof(this) && "not a label opcode");
+ }
public:
MCSymbol *getLabel() const { return Label; }
@@ -2068,6 +2181,8 @@
: MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
LSBaseSDNodeBits.AddressingMode = AM;
assert(getAddressingMode() == AM && "Value truncated");
+ assert((!MMO->isAtomic() || MMO->isVolatile()) &&
+ "use an AtomicSDNode instead for non-volatile atomics");
}
const SDValue &getOffset() const {
@@ -2492,18 +2607,6 @@
cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
}
- /// Return true if the node is a math/logic binary operator. This corresponds
- /// to the IR function of the same name.
- inline bool isBinaryOp(const SDNode *N) {
- auto Op = N->getOpcode();
- return (Op == ISD::ADD || Op == ISD::SUB || Op == ISD::MUL ||
- Op == ISD::AND || Op == ISD::OR || Op == ISD::XOR ||
- Op == ISD::SHL || Op == ISD::SRL || Op == ISD::SRA ||
- Op == ISD::SDIV || Op == ISD::UDIV || Op == ISD::SREM ||
- Op == ISD::UREM || Op == ISD::FADD || Op == ISD::FSUB ||
- Op == ISD::FMUL || Op == ISD::FDIV || Op == ISD::FREM);
- }
-
/// Attempt to match a unary predicate against a scalar/splat constant or
/// every element of a constant BUILD_VECTOR.
/// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
@@ -2514,10 +2617,11 @@
/// Attempt to match a binary predicate against a pair of scalar/splat
/// constants or every element of a pair of constant BUILD_VECTORs.
/// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
+ /// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match.
bool matchBinaryPredicate(
SDValue LHS, SDValue RHS,
std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
- bool AllowUndefs = false);
+ bool AllowUndefs = false, bool AllowTypeMismatch = false);
} // end namespace ISD
} // end namespace llvm
diff --git a/linux-x64/clang/include/llvm/CodeGen/SlotIndexes.h b/linux-x64/clang/include/llvm/CodeGen/SlotIndexes.h
index 63461d6..10ab4cc 100644
--- a/linux-x64/clang/include/llvm/CodeGen/SlotIndexes.h
+++ b/linux-x64/clang/include/llvm/CodeGen/SlotIndexes.h
@@ -308,20 +308,6 @@
using IdxMBBPair = std::pair<SlotIndex, MachineBasicBlock *>;
- inline bool operator<(SlotIndex V, const IdxMBBPair &IM) {
- return V < IM.first;
- }
-
- inline bool operator<(const IdxMBBPair &IM, SlotIndex V) {
- return IM.first < V;
- }
-
- struct Idx2MBBCompare {
- bool operator()(const IdxMBBPair &LHS, const IdxMBBPair &RHS) const {
- return LHS.first < RHS.first;
- }
- };
-
/// SlotIndexes pass.
///
/// This pass assigns indexes to each instruction.
@@ -333,10 +319,6 @@
using IndexList = ilist<IndexListEntry>;
IndexList indexList;
-#ifdef EXPENSIVE_CHECKS
- IndexList graveyardList;
-#endif // EXPENSIVE_CHECKS
-
MachineFunction *mf;
using Mi2IndexMap = DenseMap<const MachineInstr *, SlotIndex>;
@@ -382,9 +364,6 @@
/// Dump the indexes.
void dump() const;
- /// Renumber the index list, providing space for new instructions.
- void renumberIndexes();
-
/// Repair indexes after adding and removing instructions.
void repairIndexesInRange(MachineBasicBlock *MBB,
MachineBasicBlock::iterator Begin,
@@ -513,7 +492,9 @@
/// Move iterator to the next IdxMBBPair where the SlotIndex is greater or
/// equal to \p To.
MBBIndexIterator advanceMBBIndex(MBBIndexIterator I, SlotIndex To) const {
- return std::lower_bound(I, idx2MBBMap.end(), To);
+ return std::partition_point(
+ I, idx2MBBMap.end(),
+ [=](const IdxMBBPair &IM) { return IM.first < To; });
}
/// Get an iterator pointing to the IdxMBBPair with the biggest SlotIndex
@@ -549,29 +530,6 @@
return J->second;
}
- /// Returns the MBB covering the given range, or null if the range covers
- /// more than one basic block.
- MachineBasicBlock* getMBBCoveringRange(SlotIndex start, SlotIndex end) const {
-
- assert(start < end && "Backwards ranges not allowed.");
- MBBIndexIterator itr = findMBBIndex(start);
- if (itr == MBBIndexEnd()) {
- itr = std::prev(itr);
- return itr->second;
- }
-
- // Check that we don't cross the boundary into this block.
- if (itr->first < end)
- return nullptr;
-
- itr = std::prev(itr);
-
- if (itr->first <= start)
- return itr->second;
-
- return nullptr;
- }
-
/// Insert the given machine instruction into the mapping. Returns the
/// assigned index.
/// If Late is set and there are null indexes between mi's neighboring
@@ -677,33 +635,7 @@
idx2MBBMap.push_back(IdxMBBPair(startIdx, mbb));
renumberIndexes(newItr);
- llvm::sort(idx2MBBMap, Idx2MBBCompare());
- }
-
- /// Free the resources that were required to maintain a SlotIndex.
- ///
- /// Once an index is no longer needed (for instance because the instruction
- /// at that index has been moved), the resources required to maintain the
- /// index can be relinquished to reduce memory use and improve renumbering
- /// performance. Any remaining SlotIndex objects that point to the same
- /// index are left 'dangling' (much the same as a dangling pointer to a
- /// freed object) and should not be accessed, except to destruct them.
- ///
- /// Like dangling pointers, access to dangling SlotIndexes can cause
- /// painful-to-track-down bugs, especially if the memory for the index
- /// previously pointed to has been re-used. To detect dangling SlotIndex
- /// bugs, build with EXPENSIVE_CHECKS=1. This will cause "erased" indexes to
- /// be retained in a graveyard instead of being freed. Operations on indexes
- /// in the graveyard will trigger an assertion.
- void eraseIndex(SlotIndex index) {
- IndexListEntry *entry = index.listEntry();
-#ifdef EXPENSIVE_CHECKS
- indexList.remove(entry);
- graveyardList.push_back(entry);
- entry->setPoison();
-#else
- indexList.erase(entry);
-#endif
+ llvm::sort(idx2MBBMap, less_first());
}
};
diff --git a/linux-x64/clang/include/llvm/CodeGen/StackProtector.h b/linux-x64/clang/include/llvm/CodeGen/StackProtector.h
index ed52db3..2bdf442 100644
--- a/linux-x64/clang/include/llvm/CodeGen/StackProtector.h
+++ b/linux-x64/clang/include/llvm/CodeGen/StackProtector.h
@@ -61,12 +61,6 @@
/// protection when -fstack-protection is used.
unsigned SSPBufferSize = 0;
- /// VisitedPHIs - The set of PHI nodes visited when determining
- /// if a variable's reference has been taken. This set
- /// is maintained to ensure we don't visit the same PHI node multiple
- /// times.
- SmallPtrSet<const PHINode *, 16> VisitedPHIs;
-
// A prologue is generated.
bool HasPrologue = false;
diff --git a/linux-x64/clang/include/llvm/CodeGen/SwiftErrorValueTracking.h b/linux-x64/clang/include/llvm/CodeGen/SwiftErrorValueTracking.h
new file mode 100644
index 0000000..fb7a128
--- /dev/null
+++ b/linux-x64/clang/include/llvm/CodeGen/SwiftErrorValueTracking.h
@@ -0,0 +1,110 @@
+//===- SwiftErrorValueTracking.h - Track swifterror VReg vals --*- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This implements a limited mem2reg-like analysis to promote uses of function
+// arguments and allocas marked with swiftalloc from memory into virtual
+// registers tracked by this class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFTERRORVALUETRACKING_H
+#define SWIFTERRORVALUETRACKING_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/Register.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/DebugLoc.h"
+#include <functional>
+#include <type_traits>
+#include <utility>
+
+
+namespace llvm {
+ class Function;
+ class MachineBasicBlock;
+ class MachineFunction;
+ class MachineInstr;
+ class TargetInstrInfo;
+ class TargetLowering;
+
+class SwiftErrorValueTracking {
+ // Some useful objects to reduce the number of function arguments needed.
+ MachineFunction *MF;
+ const Function *Fn;
+ const TargetLowering *TLI;
+ const TargetInstrInfo *TII;
+
+ /// A map from swifterror value in a basic block to the virtual register it is
+ /// currently represented by.
+ DenseMap<std::pair<const MachineBasicBlock *, const Value *>, Register>
+ VRegDefMap;
+
+ /// A list of upward exposed vreg uses that need to be satisfied by either a
+ /// copy def or a phi node at the beginning of the basic block representing
+ /// the predecessor(s) swifterror value.
+ DenseMap<std::pair<const MachineBasicBlock *, const Value *>, Register>
+ VRegUpwardsUse;
+
+ /// A map from instructions that define/use a swifterror value to the virtual
+ /// register that represents that def/use.
+ llvm::DenseMap<PointerIntPair<const Instruction *, 1, bool>, Register>
+ VRegDefUses;
+
+ /// The swifterror argument of the current function.
+ const Value *SwiftErrorArg;
+
+ using SwiftErrorValues = SmallVector<const Value*, 1>;
+ /// A function can only have a single swifterror argument. And if it does
+ /// have a swifterror argument, it must be the first entry in
+ /// SwiftErrorVals.
+ SwiftErrorValues SwiftErrorVals;
+
+public:
+ /// Initialize data structures for specified new function.
+ void setFunction(MachineFunction &MF);
+
+ /// Get the (unique) function argument that was marked swifterror, or nullptr
+ /// if this function has no swifterror args.
+ const Value *getFunctionArg() const {
+ return SwiftErrorArg;
+ }
+
+ /// Get or create the swifterror value virtual register in
+ /// VRegDefMap for this basic block.
+ Register getOrCreateVReg(const MachineBasicBlock *, const Value *);
+
+ /// Set the swifterror virtual register in the VRegDefMap for this
+ /// basic block.
+ void setCurrentVReg(const MachineBasicBlock *MBB, const Value *, Register);
+
+ /// Get or create the swifterror value virtual register for a def of a
+ /// swifterror by an instruction.
+ Register getOrCreateVRegDefAt(const Instruction *, const MachineBasicBlock *,
+ const Value *);
+
+ /// Get or create the swifterror value virtual register for a use of a
+ /// swifterror by an instruction.
+ Register getOrCreateVRegUseAt(const Instruction *, const MachineBasicBlock *,
+ const Value *);
+
+ /// Create initial definitions of swifterror values in the entry block of the
+ /// current function.
+ bool createEntriesInEntryBlock(DebugLoc DbgLoc);
+
+ /// Propagate assigned swifterror vregs through a function, synthesizing PHI
+ /// nodes when needed to maintain consistency.
+ void propagateVRegs();
+
+ void preassignVRegs(MachineBasicBlock *MBB, BasicBlock::const_iterator Begin,
+ BasicBlock::const_iterator End);
+};
+
+}
+
+#endif
diff --git a/linux-x64/clang/include/llvm/CodeGen/SwitchLoweringUtils.h b/linux-x64/clang/include/llvm/CodeGen/SwitchLoweringUtils.h
new file mode 100644
index 0000000..62134dc
--- /dev/null
+++ b/linux-x64/clang/include/llvm/CodeGen/SwitchLoweringUtils.h
@@ -0,0 +1,297 @@
+//===- SwitchLoweringUtils.h - Switch Lowering ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_SWITCHLOWERINGUTILS_H
+#define LLVM_CODEGEN_SWITCHLOWERINGUTILS_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/SelectionDAGNodes.h"
+#include "llvm/CodeGen/TargetLowering.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/Support/BranchProbability.h"
+
+namespace llvm {
+
+class FunctionLoweringInfo;
+class MachineBasicBlock;
+
+namespace SwitchCG {
+
+enum CaseClusterKind {
+ /// A cluster of adjacent case labels with the same destination, or just one
+ /// case.
+ CC_Range,
+ /// A cluster of cases suitable for jump table lowering.
+ CC_JumpTable,
+ /// A cluster of cases suitable for bit test lowering.
+ CC_BitTests
+};
+
+/// A cluster of case labels.
+struct CaseCluster {
+ CaseClusterKind Kind;
+ const ConstantInt *Low, *High;
+ union {
+ MachineBasicBlock *MBB;
+ unsigned JTCasesIndex;
+ unsigned BTCasesIndex;
+ };
+ BranchProbability Prob;
+
+ static CaseCluster range(const ConstantInt *Low, const ConstantInt *High,
+ MachineBasicBlock *MBB, BranchProbability Prob) {
+ CaseCluster C;
+ C.Kind = CC_Range;
+ C.Low = Low;
+ C.High = High;
+ C.MBB = MBB;
+ C.Prob = Prob;
+ return C;
+ }
+
+ static CaseCluster jumpTable(const ConstantInt *Low, const ConstantInt *High,
+ unsigned JTCasesIndex, BranchProbability Prob) {
+ CaseCluster C;
+ C.Kind = CC_JumpTable;
+ C.Low = Low;
+ C.High = High;
+ C.JTCasesIndex = JTCasesIndex;
+ C.Prob = Prob;
+ return C;
+ }
+
+ static CaseCluster bitTests(const ConstantInt *Low, const ConstantInt *High,
+ unsigned BTCasesIndex, BranchProbability Prob) {
+ CaseCluster C;
+ C.Kind = CC_BitTests;
+ C.Low = Low;
+ C.High = High;
+ C.BTCasesIndex = BTCasesIndex;
+ C.Prob = Prob;
+ return C;
+ }
+};
+
+using CaseClusterVector = std::vector<CaseCluster>;
+using CaseClusterIt = CaseClusterVector::iterator;
+
+/// Sort Clusters and merge adjacent cases.
+void sortAndRangeify(CaseClusterVector &Clusters);
+
+struct CaseBits {
+ uint64_t Mask = 0;
+ MachineBasicBlock *BB = nullptr;
+ unsigned Bits = 0;
+ BranchProbability ExtraProb;
+
+ CaseBits() = default;
+ CaseBits(uint64_t mask, MachineBasicBlock *bb, unsigned bits,
+ BranchProbability Prob)
+ : Mask(mask), BB(bb), Bits(bits), ExtraProb(Prob) {}
+};
+
+using CaseBitsVector = std::vector<CaseBits>;
+
+/// This structure is used to communicate between SelectionDAGBuilder and
+/// SDISel for the code generation of additional basic blocks needed by
+/// multi-case switch statements.
+struct CaseBlock {
+ // For the GISel interface.
+ struct PredInfoPair {
+ CmpInst::Predicate Pred;
+ // Set when no comparison should be emitted.
+ bool NoCmp;
+ };
+ union {
+ // The condition code to use for the case block's setcc node.
+ // Besides the integer condition codes, this can also be SETTRUE, in which
+ // case no comparison gets emitted.
+ ISD::CondCode CC;
+ struct PredInfoPair PredInfo;
+ };
+
+ // The LHS/MHS/RHS of the comparison to emit.
+ // Emit by default LHS op RHS. MHS is used for range comparisons:
+ // If MHS is not null: (LHS <= MHS) and (MHS <= RHS).
+ const Value *CmpLHS, *CmpMHS, *CmpRHS;
+
+ // The block to branch to if the setcc is true/false.
+ MachineBasicBlock *TrueBB, *FalseBB;
+
+ // The block into which to emit the code for the setcc and branches.
+ MachineBasicBlock *ThisBB;
+
+ /// The debug location of the instruction this CaseBlock was
+ /// produced from.
+ SDLoc DL;
+ DebugLoc DbgLoc;
+
+ // Branch weights.
+ BranchProbability TrueProb, FalseProb;
+
+ // Constructor for SelectionDAG.
+ CaseBlock(ISD::CondCode cc, const Value *cmplhs, const Value *cmprhs,
+ const Value *cmpmiddle, MachineBasicBlock *truebb,
+ MachineBasicBlock *falsebb, MachineBasicBlock *me, SDLoc dl,
+ BranchProbability trueprob = BranchProbability::getUnknown(),
+ BranchProbability falseprob = BranchProbability::getUnknown())
+ : CC(cc), CmpLHS(cmplhs), CmpMHS(cmpmiddle), CmpRHS(cmprhs),
+ TrueBB(truebb), FalseBB(falsebb), ThisBB(me), DL(dl),
+ TrueProb(trueprob), FalseProb(falseprob) {}
+
+ // Constructor for GISel.
+ CaseBlock(CmpInst::Predicate pred, bool nocmp, const Value *cmplhs,
+ const Value *cmprhs, const Value *cmpmiddle,
+ MachineBasicBlock *truebb, MachineBasicBlock *falsebb,
+ MachineBasicBlock *me, DebugLoc dl,
+ BranchProbability trueprob = BranchProbability::getUnknown(),
+ BranchProbability falseprob = BranchProbability::getUnknown())
+ : PredInfo({pred, nocmp}), CmpLHS(cmplhs), CmpMHS(cmpmiddle),
+ CmpRHS(cmprhs), TrueBB(truebb), FalseBB(falsebb), ThisBB(me),
+ DbgLoc(dl), TrueProb(trueprob), FalseProb(falseprob) {}
+};
+
+struct JumpTable {
+ /// The virtual register containing the index of the jump table entry
+ /// to jump to.
+ unsigned Reg;
+ /// The JumpTableIndex for this jump table in the function.
+ unsigned JTI;
+ /// The MBB into which to emit the code for the indirect jump.
+ MachineBasicBlock *MBB;
+ /// The MBB of the default bb, which is a successor of the range
+ /// check MBB. This is when updating PHI nodes in successors.
+ MachineBasicBlock *Default;
+
+ JumpTable(unsigned R, unsigned J, MachineBasicBlock *M, MachineBasicBlock *D)
+ : Reg(R), JTI(J), MBB(M), Default(D) {}
+};
+struct JumpTableHeader {
+ APInt First;
+ APInt Last;
+ const Value *SValue;
+ MachineBasicBlock *HeaderBB;
+ bool Emitted;
+ bool OmitRangeCheck;
+
+ JumpTableHeader(APInt F, APInt L, const Value *SV, MachineBasicBlock *H,
+ bool E = false)
+ : First(std::move(F)), Last(std::move(L)), SValue(SV), HeaderBB(H),
+ Emitted(E), OmitRangeCheck(false) {}
+};
+using JumpTableBlock = std::pair<JumpTableHeader, JumpTable>;
+
+struct BitTestCase {
+ uint64_t Mask;
+ MachineBasicBlock *ThisBB;
+ MachineBasicBlock *TargetBB;
+ BranchProbability ExtraProb;
+
+ BitTestCase(uint64_t M, MachineBasicBlock *T, MachineBasicBlock *Tr,
+ BranchProbability Prob)
+ : Mask(M), ThisBB(T), TargetBB(Tr), ExtraProb(Prob) {}
+};
+
+using BitTestInfo = SmallVector<BitTestCase, 3>;
+
+struct BitTestBlock {
+ APInt First;
+ APInt Range;
+ const Value *SValue;
+ unsigned Reg;
+ MVT RegVT;
+ bool Emitted;
+ bool ContiguousRange;
+ MachineBasicBlock *Parent;
+ MachineBasicBlock *Default;
+ BitTestInfo Cases;
+ BranchProbability Prob;
+ BranchProbability DefaultProb;
+
+ BitTestBlock(APInt F, APInt R, const Value *SV, unsigned Rg, MVT RgVT, bool E,
+ bool CR, MachineBasicBlock *P, MachineBasicBlock *D,
+ BitTestInfo C, BranchProbability Pr)
+ : First(std::move(F)), Range(std::move(R)), SValue(SV), Reg(Rg),
+ RegVT(RgVT), Emitted(E), ContiguousRange(CR), Parent(P), Default(D),
+ Cases(std::move(C)), Prob(Pr) {}
+};
+
+/// Return the range of value within a range.
+uint64_t getJumpTableRange(const CaseClusterVector &Clusters, unsigned First,
+ unsigned Last);
+
+/// Return the number of cases within a range.
+uint64_t getJumpTableNumCases(const SmallVectorImpl<unsigned> &TotalCases,
+ unsigned First, unsigned Last);
+
+struct SwitchWorkListItem {
+ MachineBasicBlock *MBB;
+ CaseClusterIt FirstCluster;
+ CaseClusterIt LastCluster;
+ const ConstantInt *GE;
+ const ConstantInt *LT;
+ BranchProbability DefaultProb;
+};
+using SwitchWorkList = SmallVector<SwitchWorkListItem, 4>;
+
+class SwitchLowering {
+public:
+ SwitchLowering(FunctionLoweringInfo &funcinfo) : FuncInfo(funcinfo) {}
+
+ void init(const TargetLowering &tli, const TargetMachine &tm,
+ const DataLayout &dl) {
+ TLI = &tli;
+ TM = &tm;
+ DL = &dl;
+ }
+
+ /// Vector of CaseBlock structures used to communicate SwitchInst code
+ /// generation information.
+ std::vector<CaseBlock> SwitchCases;
+
+ /// Vector of JumpTable structures used to communicate SwitchInst code
+ /// generation information.
+ std::vector<JumpTableBlock> JTCases;
+
+ /// Vector of BitTestBlock structures used to communicate SwitchInst code
+ /// generation information.
+ std::vector<BitTestBlock> BitTestCases;
+
+ void findJumpTables(CaseClusterVector &Clusters, const SwitchInst *SI,
+ MachineBasicBlock *DefaultMBB);
+
+ bool buildJumpTable(const CaseClusterVector &Clusters, unsigned First,
+ unsigned Last, const SwitchInst *SI,
+ MachineBasicBlock *DefaultMBB, CaseCluster &JTCluster);
+
+
+ void findBitTestClusters(CaseClusterVector &Clusters, const SwitchInst *SI);
+
+ /// Build a bit test cluster from Clusters[First..Last]. Returns false if it
+ /// decides it's not a good idea.
+ bool buildBitTests(CaseClusterVector &Clusters, unsigned First, unsigned Last,
+ const SwitchInst *SI, CaseCluster &BTCluster);
+
+ virtual void addSuccessorWithProb(
+ MachineBasicBlock *Src, MachineBasicBlock *Dst,
+ BranchProbability Prob = BranchProbability::getUnknown()) = 0;
+
+ virtual ~SwitchLowering() = default;
+
+private:
+ const TargetLowering *TLI;
+ const TargetMachine *TM;
+ const DataLayout *DL;
+ FunctionLoweringInfo &FuncInfo;
+};
+
+} // namespace SwitchCG
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_SWITCHLOWERINGUTILS_H
+
diff --git a/linux-x64/clang/include/llvm/CodeGen/TargetCallingConv.h b/linux-x64/clang/include/llvm/CodeGen/TargetCallingConv.h
index f82c05d..aebeeec 100644
--- a/linux-x64/clang/include/llvm/CodeGen/TargetCallingConv.h
+++ b/linux-x64/clang/include/llvm/CodeGen/TargetCallingConv.h
@@ -45,9 +45,12 @@
unsigned IsInConsecutiveRegsLast : 1;
unsigned IsInConsecutiveRegs : 1;
unsigned IsCopyElisionCandidate : 1; ///< Argument copy elision candidate
+ unsigned IsPointer : 1;
unsigned ByValSize; ///< Byval struct size
+ unsigned PointerAddrSpace; ///< Address space of pointer argument
+
public:
ArgFlagsTy()
: IsZExt(0), IsSExt(0), IsInReg(0), IsSRet(0), IsByVal(0), IsNest(0),
@@ -55,8 +58,9 @@
IsSwiftSelf(0), IsSwiftError(0), IsHva(0), IsHvaStart(0),
IsSecArgPass(0), ByValAlign(0), OrigAlign(0),
IsInConsecutiveRegsLast(0), IsInConsecutiveRegs(0),
- IsCopyElisionCandidate(0), ByValSize(0) {
- static_assert(sizeof(*this) == 2 * sizeof(unsigned), "flags are too big");
+ IsCopyElisionCandidate(0), IsPointer(0), ByValSize(0),
+ PointerAddrSpace(0) {
+ static_assert(sizeof(*this) == 3 * sizeof(unsigned), "flags are too big");
}
bool isZExt() const { return IsZExt; }
@@ -113,6 +117,9 @@
bool isCopyElisionCandidate() const { return IsCopyElisionCandidate; }
void setCopyElisionCandidate() { IsCopyElisionCandidate = 1; }
+ bool isPointer() const { return IsPointer; }
+ void setPointer() { IsPointer = 1; }
+
unsigned getByValAlign() const { return (1U << ByValAlign) / 2; }
void setByValAlign(unsigned A) {
ByValAlign = Log2_32(A) + 1;
@@ -127,7 +134,10 @@
unsigned getByValSize() const { return ByValSize; }
void setByValSize(unsigned S) { ByValSize = S; }
- };
+
+ unsigned getPointerAddrSpace() const { return PointerAddrSpace; }
+ void setPointerAddrSpace(unsigned AS) { PointerAddrSpace = AS; }
+};
/// InputArg - This struct carries flags and type information about a
/// single incoming (formal) argument or incoming (from the perspective
diff --git a/linux-x64/clang/include/llvm/CodeGen/TargetFrameLowering.h b/linux-x64/clang/include/llvm/CodeGen/TargetFrameLowering.h
index 754ee5c..878c9ff 100644
--- a/linux-x64/clang/include/llvm/CodeGen/TargetFrameLowering.h
+++ b/linux-x64/clang/include/llvm/CodeGen/TargetFrameLowering.h
@@ -14,6 +14,7 @@
#define LLVM_CODEGEN_TARGETFRAMELOWERING_H
#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/ADT/StringSwitch.h"
#include <utility>
#include <vector>
@@ -23,6 +24,14 @@
class MachineFunction;
class RegScavenger;
+namespace TargetStackID {
+ enum Value {
+ Default = 0,
+ SGPRSpill = 1,
+ NoAlloc = 255
+ };
+}
+
/// Information about stack frame layout on the target. It holds the direction
/// of stack growth, the known stack alignment on entry to each function, and
/// the offset to the locals area.
@@ -345,6 +354,16 @@
return true;
}
+ virtual bool isSupportedStackID(TargetStackID::Value ID) const {
+ switch (ID) {
+ default:
+ return false;
+ case TargetStackID::Default:
+ case TargetStackID::NoAlloc:
+ return true;
+ }
+ }
+
/// Check if given function is safe for not having callee saved registers.
/// This is used when interprocedural register allocation is enabled.
static bool isSafeForNoCSROpt(const Function &F) {
diff --git a/linux-x64/clang/include/llvm/CodeGen/TargetInstrInfo.h b/linux-x64/clang/include/llvm/CodeGen/TargetInstrInfo.h
index b732be6..314bb72 100644
--- a/linux-x64/clang/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/linux-x64/clang/include/llvm/CodeGen/TargetInstrInfo.h
@@ -26,6 +26,7 @@
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineOutliner.h"
#include "llvm/CodeGen/PseudoSourceValue.h"
+#include "llvm/CodeGen/VirtRegMap.h"
#include "llvm/MC/MCInstrInfo.h"
#include "llvm/Support/BranchProbability.h"
#include "llvm/Support/ErrorHandling.h"
@@ -59,6 +60,8 @@
template <class T> class SmallVectorImpl;
+using ParamLoadedValue = std::pair<const MachineOperand*, DIExpression*>;
+
//---------------------------------------------------------------------------
///
/// TargetInstrInfo - Interface to description of machine instruction set
@@ -80,6 +83,7 @@
/// Given a machine instruction descriptor, returns the register
/// class constraint for OpNum, or NULL.
+ virtual
const TargetRegisterClass *getRegClass(const MCInstrDesc &MCID, unsigned OpNum,
const TargetRegisterInfo *TRI,
const MachineFunction &MF) const;
@@ -669,8 +673,9 @@
/// is finished. Return the value/register of the new loop count. We need
/// this function when peeling off one or more iterations of a loop. This
/// function assumes the nth iteration is peeled first.
- virtual unsigned reduceLoopCount(MachineBasicBlock &MBB, MachineInstr *IndVar,
- MachineInstr &Cmp,
+ virtual unsigned reduceLoopCount(MachineBasicBlock &MBB,
+ MachineBasicBlock &PreHeader,
+ MachineInstr *IndVar, MachineInstr &Cmp,
SmallVectorImpl<MachineOperand> &Cond,
SmallVectorImpl<MachineInstr *> &PrevInsts,
unsigned Iter, unsigned MaxIter) const {
@@ -932,9 +937,12 @@
/// operand folded, otherwise NULL is returned.
/// The new instruction is inserted before MI, and the client is responsible
/// for removing the old instruction.
+ /// If VRM is passed, the assigned physregs can be inspected by target to
+ /// decide on using an opcode (note that those assignments can still change).
MachineInstr *foldMemoryOperand(MachineInstr &MI, ArrayRef<unsigned> Ops,
int FI,
- LiveIntervals *LIS = nullptr) const;
+ LiveIntervals *LIS = nullptr,
+ VirtRegMap *VRM = nullptr) const;
/// Same as the previous version except it allows folding of any load and
/// store from / to any address, not just from a specific stack slot.
@@ -1024,7 +1032,8 @@
foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
ArrayRef<unsigned> Ops,
MachineBasicBlock::iterator InsertPt, int FrameIndex,
- LiveIntervals *LIS = nullptr) const {
+ LiveIntervals *LIS = nullptr,
+ VirtRegMap *VRM = nullptr) const {
return nullptr;
}
@@ -1144,8 +1153,9 @@
/// Get the base operand and byte offset of an instruction that reads/writes
/// memory.
- virtual bool getMemOperandWithOffset(MachineInstr &MI,
- MachineOperand *&BaseOp, int64_t &Offset,
+ virtual bool getMemOperandWithOffset(const MachineInstr &MI,
+ const MachineOperand *&BaseOp,
+ int64_t &Offset,
const TargetRegisterInfo *TRI) const {
return false;
}
@@ -1170,8 +1180,8 @@
/// or
/// DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
/// to TargetPassConfig::createMachineScheduler() to have an effect.
- virtual bool shouldClusterMemOps(MachineOperand &BaseOp1,
- MachineOperand &BaseOp2,
+ virtual bool shouldClusterMemOps(const MachineOperand &BaseOp1,
+ const MachineOperand &BaseOp2,
unsigned NumLoads) const {
llvm_unreachable("target did not implement shouldClusterMemOps()");
}
@@ -1259,8 +1269,9 @@
/// Measure the specified inline asm to determine an approximation of its
/// length.
- virtual unsigned getInlineAsmLength(const char *Str,
- const MCAsmInfo &MAI) const;
+ virtual unsigned getInlineAsmLength(
+ const char *Str, const MCAsmInfo &MAI,
+ const TargetSubtargetInfo *STI = nullptr) const;
/// Allocate and return a hazard recognizer to use for this target when
/// scheduling the machine instructions before register allocation.
@@ -1548,7 +1559,8 @@
/// See also MachineInstr::mayAlias, which is implemented on top of this
/// function.
virtual bool
- areMemAccessesTriviallyDisjoint(MachineInstr &MIa, MachineInstr &MIb,
+ areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
+ const MachineInstr &MIb,
AliasAnalysis *AA = nullptr) const {
assert((MIa.mayLoad() || MIa.mayStore()) &&
"MIa must load from or modify a memory location");
@@ -1681,6 +1693,10 @@
return false;
}
+ /// Produce RHS description of parameter's loading instruction \p MI.
+ virtual Optional<ParamLoadedValue>
+ describeLoadedValue(const MachineInstr &MI) const;
+
private:
unsigned CallFrameSetupOpcode, CallFrameDestroyOpcode;
unsigned CatchRetOpcode;
diff --git a/linux-x64/clang/include/llvm/CodeGen/TargetLowering.h b/linux-x64/clang/include/llvm/CodeGen/TargetLowering.h
index fa71024..d5cca60 100644
--- a/linux-x64/clang/include/llvm/CodeGen/TargetLowering.h
+++ b/linux-x64/clang/include/llvm/CodeGen/TargetLowering.h
@@ -188,6 +188,7 @@
bool IsSwiftSelf : 1;
bool IsSwiftError : 1;
uint16_t Alignment = 0;
+ Type *ByValType = nullptr;
ArgListEntry()
: IsSExt(false), IsZExt(false), IsInReg(false), IsSRet(false),
@@ -238,7 +239,14 @@
/// Return the pointer type for the given address space, defaults to
/// the pointer type from the data layout.
/// FIXME: The default needs to be removed once all the code is updated.
- MVT getPointerTy(const DataLayout &DL, uint32_t AS = 0) const {
+ virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS = 0) const {
+ return MVT::getIntegerVT(DL.getPointerSizeInBits(AS));
+ }
+
+ /// Return the in-memory pointer type for the given address space, defaults to
+ /// the pointer type from the data layout. FIXME: The default needs to be
+ /// removed once all the code is updated.
+ MVT getPointerMemTy(const DataLayout &DL, uint32_t AS = 0) const {
return MVT::getIntegerVT(DL.getPointerSizeInBits(AS));
}
@@ -294,6 +302,9 @@
// The default action for one element vectors is to scalarize
if (VT.getVectorNumElements() == 1)
return TypeScalarizeVector;
+ // The default action for an odd-width vector is to widen.
+ if (!VT.isPow2VectorType())
+ return TypeWidenVector;
// The default action for other vectors is to promote
return TypePromoteInteger;
}
@@ -390,8 +401,9 @@
/// efficiently, casting the load to a smaller vector of larger types and
/// loading is more efficient, however, this can be undone by optimizations in
/// dag combiner.
- virtual bool isLoadBitCastBeneficial(EVT LoadVT,
- EVT BitcastVT) const {
+ virtual bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT,
+ const SelectionDAG &DAG,
+ const MachineMemOperand &MMO) const {
// Don't do if we could do an indexed load on the original type, but not on
// the new one.
if (!LoadVT.isSimple() || !BitcastVT.isSimple())
@@ -405,14 +417,18 @@
getTypeToPromoteTo(ISD::LOAD, LoadMVT) == BitcastVT.getSimpleVT())
return false;
- return true;
+ bool Fast = false;
+ return allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), BitcastVT,
+ MMO, &Fast) && Fast;
}
/// Return true if the following transform is beneficial:
/// (store (y (conv x)), y*)) -> (store x, (x*))
- virtual bool isStoreBitCastBeneficial(EVT StoreVT, EVT BitcastVT) const {
+ virtual bool isStoreBitCastBeneficial(EVT StoreVT, EVT BitcastVT,
+ const SelectionDAG &DAG,
+ const MachineMemOperand &MMO) const {
// Default to the same logic as loads.
- return isLoadBitCastBeneficial(StoreVT, BitcastVT);
+ return isLoadBitCastBeneficial(StoreVT, BitcastVT, DAG, MMO);
}
/// Return true if it is expected to be cheaper to do a store of a non-zero
@@ -424,10 +440,12 @@
return false;
}
- /// Allow store merging after legalization in addition to before legalization.
- /// This may catch stores that do not exist earlier (eg, stores created from
- /// intrinsics).
- virtual bool mergeStoresAfterLegalization() const { return true; }
+ /// Allow store merging for the specified type after legalization in addition
+ /// to before legalization. This may transform stores that do not exist
+ /// earlier (for example, stores created from intrinsics).
+ virtual bool mergeStoresAfterLegalization(EVT MemVT) const {
+ return true;
+ }
/// Returns if it's reasonable to merge stores to MemVT size.
virtual bool canMergeStoresTo(unsigned AS, EVT MemVT,
@@ -524,13 +542,22 @@
/// There are two ways to clear extreme bits (either low or high):
/// Mask: x & (-1 << y) (the instcombine canonical form)
/// Shifts: x >> y << y
- /// Return true if the variant with 2 shifts is preferred.
+ /// Return true if the variant with 2 variable shifts is preferred.
/// Return false if there is no preference.
- virtual bool preferShiftsToClearExtremeBits(SDValue X) const {
+ virtual bool shouldFoldMaskToVariableShiftPair(SDValue X) const {
// By default, let's assume that no one prefers shifts.
return false;
}
+ /// Return true if it is profitable to fold a pair of shifts into a mask.
+ /// This is usually true on most targets. But some targets, like Thumb1,
+ /// have immediate shift instructions, but no immediate "and" instruction;
+ /// this makes the fold unprofitable.
+ virtual bool shouldFoldConstantShiftPairToMask(const SDNode *N,
+ CombineLevel Level) const {
+ return true;
+ }
+
/// Should we tranform the IR-optimal check for whether given truncation
/// down into KeptBits would be truncating or not:
/// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
@@ -544,6 +571,16 @@
return false;
}
+ /// These two forms are equivalent:
+ /// sub %y, (xor %x, -1)
+ /// add (add %x, 1), %y
+ /// The variant with two add's is IR-canonical.
+ /// Some targets may prefer one to the other.
+ virtual bool preferIncOfAddToSubOfNot(EVT VT) const {
+ // By default, let's assume that everyone prefers the form with two add's.
+ return true;
+ }
+
/// Return true if the target wants to use the optimization that
/// turns ext(promotableInst1(...(promotableInstN(load)))) into
/// promotedInst1(...(promotedInstN(ext(load)))).
@@ -563,11 +600,6 @@
return false;
}
- /// Return true if target supports floating point exceptions.
- bool hasFloatingPointExceptions() const {
- return HasFloatingPointExceptions;
- }
-
/// Return true if target always beneficiates from combining into FMA for a
/// given value type. This must typically return false on targets where FMA
/// takes more cycles to execute than FADD.
@@ -622,12 +654,21 @@
/// Return the register class that should be used for the specified value
/// type.
- virtual const TargetRegisterClass *getRegClassFor(MVT VT) const {
+ virtual const TargetRegisterClass *getRegClassFor(MVT VT, bool isDivergent = false) const {
+ (void)isDivergent;
const TargetRegisterClass *RC = RegClassForVT[VT.SimpleTy];
assert(RC && "This value type is not natively supported!");
return RC;
}
+ /// Allows target to decide about the register class of the
+ /// specific value that is live outside the defining block.
+ /// Returns true if the value needs uniform register class.
+ virtual bool requiresUniformRegister(MachineFunction &MF,
+ const Value *) const {
+ return false;
+ }
+
/// Return the 'representative' register class for the specified value
/// type.
///
@@ -778,7 +819,8 @@
/// Returns true if the target can instruction select the specified FP
/// immediate natively. If false, the legalizer will materialize the FP
/// immediate as a load from a constant pool.
- virtual bool isFPImmLegal(const APFloat &/*Imm*/, EVT /*VT*/) const {
+ virtual bool isFPImmLegal(const APFloat & /*Imm*/, EVT /*VT*/,
+ bool ForCodeSize = false) const {
return false;
}
@@ -840,6 +882,7 @@
default:
llvm_unreachable("Unexpected fixed point operation.");
case ISD::SMULFIX:
+ case ISD::SMULFIXSAT:
case ISD::UMULFIX:
Supported = isSupportedFixedPointOperation(Op, VT, Scale);
break;
@@ -876,6 +919,8 @@
case ISD::STRICT_FFLOOR: EqOpc = ISD::FFLOOR; break;
case ISD::STRICT_FROUND: EqOpc = ISD::FROUND; break;
case ISD::STRICT_FTRUNC: EqOpc = ISD::FTRUNC; break;
+ case ISD::STRICT_FP_ROUND: EqOpc = ISD::FP_ROUND; break;
+ case ISD::STRICT_FP_EXTEND: EqOpc = ISD::FP_EXTEND; break;
}
auto Action = getOperationAction(EqOpc, VT);
@@ -942,21 +987,20 @@
/// Return true if lowering to a jump table is suitable for a set of case
/// clusters which may contain \p NumCases cases, \p Range range of values.
- /// FIXME: This function check the maximum table size and density, but the
- /// minimum size is not checked. It would be nice if the minimum size is
- /// also combined within this function. Currently, the minimum size check is
- /// performed in findJumpTable() in SelectionDAGBuiler and
- /// getEstimatedNumberOfCaseClusters() in BasicTTIImpl.
virtual bool isSuitableForJumpTable(const SwitchInst *SI, uint64_t NumCases,
uint64_t Range) const {
- const bool OptForSize = SI->getParent()->getParent()->optForSize();
+ // FIXME: This function check the maximum table size and density, but the
+ // minimum size is not checked. It would be nice if the minimum size is
+ // also combined within this function. Currently, the minimum size check is
+ // performed in findJumpTable() in SelectionDAGBuiler and
+ // getEstimatedNumberOfCaseClusters() in BasicTTIImpl.
+ const bool OptForSize = SI->getParent()->getParent()->hasOptSize();
const unsigned MinDensity = getMinimumJumpTableDensity(OptForSize);
- const unsigned MaxJumpTableSize =
- OptForSize || getMaximumJumpTableSize() == 0
- ? UINT_MAX
- : getMaximumJumpTableSize();
- // Check whether a range of clusters is dense enough for a jump table.
- if (Range <= MaxJumpTableSize &&
+ const unsigned MaxJumpTableSize = getMaximumJumpTableSize();
+
+ // Check whether the number of cases is small enough and
+ // the range is dense enough for a jump table.
+ if ((OptForSize || Range <= MaxJumpTableSize) &&
(NumCases * 100 >= Range * MinDensity)) {
return true;
}
@@ -1151,24 +1195,42 @@
EVT getValueType(const DataLayout &DL, Type *Ty,
bool AllowUnknown = false) const {
// Lower scalar pointers to native pointer types.
- if (PointerType *PTy = dyn_cast<PointerType>(Ty))
+ if (auto *PTy = dyn_cast<PointerType>(Ty))
return getPointerTy(DL, PTy->getAddressSpace());
- if (Ty->isVectorTy()) {
- VectorType *VTy = cast<VectorType>(Ty);
- Type *Elm = VTy->getElementType();
+ if (auto *VTy = dyn_cast<VectorType>(Ty)) {
+ Type *EltTy = VTy->getElementType();
// Lower vectors of pointers to native pointer types.
+ if (auto *PTy = dyn_cast<PointerType>(EltTy)) {
+ EVT PointerTy(getPointerTy(DL, PTy->getAddressSpace()));
+ EltTy = PointerTy.getTypeForEVT(Ty->getContext());
+ }
+ return EVT::getVectorVT(Ty->getContext(), EVT::getEVT(EltTy, false),
+ VTy->getNumElements());
+ }
+
+ return EVT::getEVT(Ty, AllowUnknown);
+ }
+
+ EVT getMemValueType(const DataLayout &DL, Type *Ty,
+ bool AllowUnknown = false) const {
+ // Lower scalar pointers to native pointer types.
+ if (PointerType *PTy = dyn_cast<PointerType>(Ty))
+ return getPointerMemTy(DL, PTy->getAddressSpace());
+ else if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {
+ Type *Elm = VTy->getElementType();
if (PointerType *PT = dyn_cast<PointerType>(Elm)) {
- EVT PointerTy(getPointerTy(DL, PT->getAddressSpace()));
+ EVT PointerTy(getPointerMemTy(DL, PT->getAddressSpace()));
Elm = PointerTy.getTypeForEVT(Ty->getContext());
}
-
return EVT::getVectorVT(Ty->getContext(), EVT::getEVT(Elm, false),
VTy->getNumElements());
}
- return EVT::getEVT(Ty, AllowUnknown);
+
+ return getValueType(DL, Ty, AllowUnknown);
}
+
/// Return the MVT corresponding to this LLVM type. See getValueType.
MVT getSimpleValueType(const DataLayout &DL, Type *Ty,
bool AllowUnknown = false) const {
@@ -1338,18 +1400,6 @@
return OptSize ? MaxLoadsPerMemcmpOptSize : MaxLoadsPerMemcmp;
}
- /// For memcmp expansion when the memcmp result is only compared equal or
- /// not-equal to 0, allow up to this number of load pairs per block. As an
- /// example, this may allow 'memcmp(a, b, 3) == 0' in a single block:
- /// a0 = load2bytes &a[0]
- /// b0 = load2bytes &b[0]
- /// a2 = load1byte &a[2]
- /// b2 = load1byte &b[2]
- /// r = cmp eq (a0 ^ b0 | a2 ^ b2), 0
- virtual unsigned getMemcmpEqZeroLoadsPerBlock() const {
- return 1;
- }
-
/// Get maximum # of store operations permitted for llvm.memmove
///
/// This function returns the maximum number of store operations permitted
@@ -1369,10 +1419,10 @@
/// copy/move/set is converted to a sequence of store operations. Its use
/// helps to ensure that such replacements don't generate code that causes an
/// alignment error (trap) on the target machine.
- virtual bool allowsMisalignedMemoryAccesses(EVT,
- unsigned AddrSpace = 0,
- unsigned Align = 1,
- bool * /*Fast*/ = nullptr) const {
+ virtual bool allowsMisalignedMemoryAccesses(
+ EVT, unsigned AddrSpace = 0, unsigned Align = 1,
+ MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
+ bool * /*Fast*/ = nullptr) const {
return false;
}
@@ -1380,8 +1430,18 @@
/// given address space and alignment. If the access is allowed, the optional
/// final parameter returns if the access is also fast (as defined by the
/// target).
+ bool
+ allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT,
+ unsigned AddrSpace = 0, unsigned Alignment = 1,
+ MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
+ bool *Fast = nullptr) const;
+
+ /// Return true if the target supports a memory access of this type for the
+ /// given MachineMemOperand. If the access is allowed, the optional
+ /// final parameter returns if the access is also fast (as defined by the
+ /// target).
bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT,
- unsigned AddrSpace = 0, unsigned Alignment = 1,
+ const MachineMemOperand &MMO,
bool *Fast = nullptr) const;
/// Returns the target specific optimal type for load and store operations as
@@ -1395,12 +1455,11 @@
/// zero. 'MemcpyStrSrc' indicates whether the memcpy source is constant so it
/// does not need to be loaded. It returns EVT::Other if the type should be
/// determined using generic target-independent logic.
- virtual EVT getOptimalMemOpType(uint64_t /*Size*/,
- unsigned /*DstAlign*/, unsigned /*SrcAlign*/,
- bool /*IsMemset*/,
- bool /*ZeroMemset*/,
- bool /*MemcpyStrSrc*/,
- MachineFunction &/*MF*/) const {
+ virtual EVT
+ getOptimalMemOpType(uint64_t /*Size*/, unsigned /*DstAlign*/,
+ unsigned /*SrcAlign*/, bool /*IsMemset*/,
+ bool /*ZeroMemset*/, bool /*MemcpyStrSrc*/,
+ const AttributeList & /*FuncAttributes*/) const {
return MVT::Other;
}
@@ -1548,8 +1607,9 @@
}
/// Returns true if a cast from SrcAS to DestAS is "cheap", such that e.g. we
- /// are happy to sink it into basic blocks.
- virtual bool isCheapAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const {
+ /// are happy to sink it into basic blocks. A cast may be free, but not
+ /// necessarily a no-op. e.g. a free truncate from a 64-bit to 32-bit pointer.
+ virtual bool isFreeAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const {
return isNoopAddrSpaceCast(SrcAS, DestAS);
}
@@ -1879,12 +1939,6 @@
/// control.
void setJumpIsExpensive(bool isExpensive = true);
- /// Tells the code generator that this target supports floating point
- /// exceptions and cares about preserving floating point exception behavior.
- void setHasFloatingPointExceptions(bool FPExceptions = true) {
- HasFloatingPointExceptions = FPExceptions;
- }
-
/// Tells the code generator which bitwidths to bypass.
void addBypassSlowDiv(unsigned int SlowBitWidth, unsigned int FastBitWidth) {
BypassSlowDivWidths[SlowBitWidth] = FastBitWidth;
@@ -2173,6 +2227,8 @@
case ISD::UADDSAT:
case ISD::FMINNUM:
case ISD::FMAXNUM:
+ case ISD::FMINNUM_IEEE:
+ case ISD::FMAXNUM_IEEE:
case ISD::FMINIMUM:
case ISD::FMAXIMUM:
return true;
@@ -2180,6 +2236,30 @@
}
}
+ /// Return true if the node is a math/logic binary operator.
+ virtual bool isBinOp(unsigned Opcode) const {
+ // A commutative binop must be a binop.
+ if (isCommutativeBinOp(Opcode))
+ return true;
+ // These are non-commutative binops.
+ switch (Opcode) {
+ case ISD::SUB:
+ case ISD::SHL:
+ case ISD::SRL:
+ case ISD::SRA:
+ case ISD::SDIV:
+ case ISD::UDIV:
+ case ISD::SREM:
+ case ISD::UREM:
+ case ISD::FSUB:
+ case ISD::FDIV:
+ case ISD::FREM:
+ return true;
+ default:
+ return false;
+ }
+ }
+
/// Return true if it's free to truncate a value of type FromTy to type
/// ToTy. e.g. On x86 it's free to truncate a i32 value in register EAX to i16
/// by referencing its sub-register AX.
@@ -2439,6 +2519,31 @@
return false;
}
+ /// Return true if extraction of a scalar element from the given vector type
+ /// at the given index is cheap. For example, if scalar operations occur on
+ /// the same register file as vector operations, then an extract element may
+ /// be a sub-register rename rather than an actual instruction.
+ virtual bool isExtractVecEltCheap(EVT VT, unsigned Index) const {
+ return false;
+ }
+
+ /// Try to convert math with an overflow comparison into the corresponding DAG
+ /// node operation. Targets may want to override this independently of whether
+ /// the operation is legal/custom for the given type because it may obscure
+ /// matching of other patterns.
+ virtual bool shouldFormOverflowOp(unsigned Opcode, EVT VT) const {
+ // TODO: The default logic is inherited from code in CodeGenPrepare.
+ // The opcode should not make a difference by default?
+ if (Opcode != ISD::UADDO)
+ return false;
+
+ // Allow the transform as long as we have an integer type that is not
+ // obviously illegal and unsupported.
+ if (VT.isVector())
+ return false;
+ return VT.isSimple() || !isOperationExpand(Opcode, VT);
+ }
+
// Return true if it is profitable to use a scalar input to a BUILD_VECTOR
// even if the vector itself has multiple uses.
virtual bool aggressivelyPreferBuildVectorSources(EVT VecVT) const {
@@ -2519,10 +2624,6 @@
/// predication.
bool JumpIsExpensive;
- /// Whether the target supports or cares about preserving floating point
- /// exception behavior.
- bool HasFloatingPointExceptions;
-
/// This target prefers to use _setjmp to implement llvm.setjmp.
///
/// Defaults to false.
@@ -2858,11 +2959,10 @@
/// Returns a pair of (return value, chain).
/// It is an error to pass RTLIB::UNKNOWN_LIBCALL as \p LC.
- std::pair<SDValue, SDValue> makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC,
- EVT RetVT, ArrayRef<SDValue> Ops,
- bool isSigned, const SDLoc &dl,
- bool doesNotReturn = false,
- bool isReturnValueUsed = true) const;
+ std::pair<SDValue, SDValue> makeLibCall(
+ SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef<SDValue> Ops,
+ bool isSigned, const SDLoc &dl, bool doesNotReturn = false,
+ bool isReturnValueUsed = true, bool isPostTypeLegalization = false) const;
/// Check whether parameters to a call that are passed in callee saved
/// registers are the same as from the calling function. This needs to be
@@ -2900,6 +3000,20 @@
}
};
+ /// Determines the optimal series of memory ops to replace the memset / memcpy.
+ /// Return true if the number of memory ops is below the threshold (Limit).
+ /// It returns the types of the sequence of memory ops to perform
+ /// memset / memcpy by reference.
+ bool findOptimalMemOpLowering(std::vector<EVT> &MemOps,
+ unsigned Limit, uint64_t Size,
+ unsigned DstAlign, unsigned SrcAlign,
+ bool IsMemset,
+ bool ZeroMemset,
+ bool MemcpyStrSrc,
+ bool AllowOverlap,
+ unsigned DstAS, unsigned SrcAS,
+ const AttributeList &FuncAttributes) const;
+
/// Check to see if the specified operand of the specified instruction is a
/// constant integer. If so, check to see if there are any bits set in the
/// constant that are not demanded. If so, shrink the constant and return
@@ -3025,6 +3139,10 @@
TargetLoweringOpt &TLO,
unsigned Depth = 0) const;
+ /// This method returns the constant pool value that will be loaded by LD.
+ /// NOTE: You must check for implicit extensions of the constant by LD.
+ virtual const Constant *getTargetConstantFromLoad(LoadSDNode *LD) const;
+
/// If \p SNaN is false, \returns true if \p Op is known to never be any
/// NaN. If \p sNaN is true, returns if \p Op is known to never be a signaling
/// NaN.
@@ -3112,15 +3230,6 @@
return true;
}
- /// Return true if it is profitable to fold a pair of shifts into a mask.
- /// This is usually true on most targets. But some targets, like Thumb1,
- /// have immediate shift instructions, but no immediate "and" instruction;
- /// this makes the fold unprofitable.
- virtual bool shouldFoldShiftPairToMask(const SDNode *N,
- CombineLevel Level) const {
- return true;
- }
-
// Return true if it is profitable to combine a BUILD_VECTOR with a stride-pattern
// to a shuffle and a truncate.
// Example of such a combine:
@@ -3454,6 +3563,15 @@
return false;
}
+ /// For most targets, an LLVM type must be broken down into multiple
+ /// smaller types. Usually the halves are ordered according to the endianness
+ /// but for some platform that would break. So this method will default to
+ /// matching the endianness but can be overridden.
+ virtual bool
+ shouldSplitFunctionArgumentsAsLittleEndian(const DataLayout &DL) const {
+ return DL.isLittleEndian();
+ }
+
/// Returns a 0 terminated array of registers that can be safely used as
/// scratch registers.
virtual const MCPhysReg *getScratchRegisters(CallingConv::ID CC) const {
@@ -3663,7 +3781,7 @@
SelectionDAG &DAG) const;
// Lower custom output constraints. If invalid, return SDValue().
- virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue *Flag,
+ virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag,
SDLoc DL,
const AsmOperandInfo &OpInfo,
SelectionDAG &DAG) const;
@@ -3872,6 +3990,25 @@
/// integers as its arguments.
SDValue expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const;
+ /// Method for building the DAG expansion of ISD::U(ADD|SUB)O. Expansion
+ /// always suceeds and populates the Result and Overflow arguments.
+ void expandUADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow,
+ SelectionDAG &DAG) const;
+
+ /// Method for building the DAG expansion of ISD::S(ADD|SUB)O. Expansion
+ /// always suceeds and populates the Result and Overflow arguments.
+ void expandSADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow,
+ SelectionDAG &DAG) const;
+
+ /// Method for building the DAG expansion of ISD::[US]MULO. Returns whether
+ /// expansion was successful and populates the Result and Overflow arguments.
+ bool expandMULO(SDNode *Node, SDValue &Result, SDValue &Overflow,
+ SelectionDAG &DAG) const;
+
+ /// Expand a VECREDUCE_* into an explicit calculation. If Count is specified,
+ /// only the first Count elements of the vector are used.
+ SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const;
+
//===--------------------------------------------------------------------===//
// Instruction Emitting Hooks
//
@@ -3932,6 +4069,14 @@
SDValue N1, ISD::CondCode Cond,
DAGCombinerInfo &DCI,
const SDLoc &DL) const;
+
+ SDValue prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
+ SDValue CompTargetNode, ISD::CondCode Cond,
+ DAGCombinerInfo &DCI, const SDLoc &DL,
+ SmallVectorImpl<SDNode *> &Created) const;
+ SDValue buildUREMEqFold(EVT SETCCVT, SDValue REMNode, SDValue CompTargetNode,
+ ISD::CondCode Cond, DAGCombinerInfo &DCI,
+ const SDLoc &DL) const;
};
/// Given an LLVM IR type and return type attributes, compute the return value
diff --git a/linux-x64/clang/include/llvm/CodeGen/TargetPassConfig.h b/linux-x64/clang/include/llvm/CodeGen/TargetPassConfig.h
index 1def50d..0bd82aa 100644
--- a/linux-x64/clang/include/llvm/CodeGen/TargetPassConfig.h
+++ b/linux-x64/clang/include/llvm/CodeGen/TargetPassConfig.h
@@ -24,6 +24,7 @@
struct MachineSchedContext;
class PassConfigImpl;
class ScheduleDAGInstrs;
+class CSEConfigBase;
// The old pass manager infrastructure is hidden in a legacy namespace now.
namespace legacy {
@@ -319,6 +320,9 @@
/// By default, it's enabled for non O0 levels.
virtual bool isGISelCSEEnabled() const;
+ /// Returns the CSEConfig object to use for the current optimization level.
+ virtual std::unique_ptr<CSEConfigBase> getCSEConfig() const;
+
protected:
// Helper to verify the analysis is really immutable.
void setOpt(bool &Opt, bool Val);
@@ -360,11 +364,11 @@
/// addFastRegAlloc - Add the minimum set of target-independent passes that
/// are required for fast register allocation.
- virtual void addFastRegAlloc(FunctionPass *RegAllocPass);
+ virtual void addFastRegAlloc();
/// addOptimizedRegAlloc - Add passes related to register allocation.
/// LLVMTargetMachine provides standard regalloc passes for most targets.
- virtual void addOptimizedRegAlloc(FunctionPass *RegAllocPass);
+ virtual void addOptimizedRegAlloc();
/// addPreRewrite - Add passes to the optimized register allocation pipeline
/// after register allocation is complete, but before virtual registers are
@@ -374,10 +378,18 @@
/// after RABasic or RAGreedy, they should take advantage of LiveRegMatrix.
/// When these passes run, VirtRegMap contains legal physreg assignments for
/// all virtual registers.
+ ///
+ /// Note if the target overloads addRegAssignAndRewriteOptimized, this may not
+ /// be honored. This is also not generally used for the the fast variant,
+ /// where the allocation and rewriting are done in one pass.
virtual bool addPreRewrite() {
return false;
}
+ /// Add passes to be run immediately after virtual registers are rewritten
+ /// to physical registers.
+ virtual void addPostRewrite() { }
+
/// This method may be implemented by targets that want to run passes after
/// register allocation pass pipeline but before prolog-epilog insertion.
virtual void addPostRegAlloc() { }
@@ -431,7 +443,12 @@
/// addMachinePasses helper to create the target-selected or overriden
/// regalloc pass.
- FunctionPass *createRegAllocPass(bool Optimized);
+ virtual FunctionPass *createRegAllocPass(bool Optimized);
+
+ /// Add core register alloator passes which do the actual register assignment
+ /// and rewriting. \returns true if any passes were added.
+ virtual bool addRegAssignmentFast();
+ virtual bool addRegAssignmentOptimized();
};
} // end namespace llvm
diff --git a/linux-x64/clang/include/llvm/CodeGen/TargetRegisterInfo.h b/linux-x64/clang/include/llvm/CodeGen/TargetRegisterInfo.h
index 5ed1e44..9a3ab47 100644
--- a/linux-x64/clang/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/linux-x64/clang/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -520,15 +520,18 @@
/// function. Used by MachineRegisterInfo::isConstantPhysReg().
virtual bool isConstantPhysReg(unsigned PhysReg) const { return false; }
+ /// Returns true if the register class is considered divergent.
+ virtual bool isDivergentRegClass(const TargetRegisterClass *RC) const {
+ return false;
+ }
+
/// Physical registers that may be modified within a function but are
/// guaranteed to be restored before any uses. This is useful for targets that
/// have call sequences where a GOT register may be updated by the caller
/// prior to a call and is guaranteed to be restored (also by the caller)
/// after the call.
virtual bool isCallerPreservedPhysReg(unsigned PhysReg,
- const MachineFunction &MF) const {
- return false;
- }
+ const MachineFunction &MF) const;
/// Prior to adding the live-out mask to a stackmap or patchpoint
/// instruction, provide the target the opportunity to adjust it (mainly to
@@ -985,7 +988,7 @@
/// getFrameRegister - This method should return the register used as a base
/// for values allocated in the current stack frame.
- virtual unsigned getFrameRegister(const MachineFunction &MF) const = 0;
+ virtual Register getFrameRegister(const MachineFunction &MF) const = 0;
/// Mark a register and all its aliases as reserved in the given set.
void markSuperRegs(BitVector &RegisterSet, unsigned Reg) const;
diff --git a/linux-x64/clang/include/llvm/CodeGen/TargetSubtargetInfo.h b/linux-x64/clang/include/llvm/CodeGen/TargetSubtargetInfo.h
index bf0e9b2..037fc3e 100644
--- a/linux-x64/clang/include/llvm/CodeGen/TargetSubtargetInfo.h
+++ b/linux-x64/clang/include/llvm/CodeGen/TargetSubtargetInfo.h
@@ -42,6 +42,7 @@
class SDep;
class SelectionDAGTargetInfo;
struct SubtargetFeatureKV;
+struct SubtargetSubTypeKV;
struct SubtargetInfoKV;
class SUnit;
class TargetFrameLowering;
@@ -62,8 +63,7 @@
protected: // Can only create subclasses...
TargetSubtargetInfo(const Triple &TT, StringRef CPU, StringRef FS,
ArrayRef<SubtargetFeatureKV> PF,
- ArrayRef<SubtargetFeatureKV> PD,
- const SubtargetInfoKV *ProcSched,
+ ArrayRef<SubtargetSubTypeKV> PD,
const MCWriteProcResEntry *WPR,
const MCWriteLatencyEntry *WL,
const MCReadAdvanceEntry *RA, const InstrStage *IS,
@@ -193,6 +193,9 @@
/// for preRA scheduling with the source level scheduler.
virtual bool enableMachineSchedDefaultSched() const { return true; }
+ /// True if the subtarget should run MachinePipeliner
+ virtual bool enableMachinePipeliner() const { return true; };
+
/// True if the subtarget should enable joining global copies.
///
/// By default this is enabled if the machine scheduler is enabled, but
@@ -246,6 +249,10 @@
std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations) const {
}
+ /// Default to DFA for resource management, return false when target will use
+ /// ProcResource in InstrSchedModel instead.
+ virtual bool useDFAforSMS() const { return true; }
+
// For use with PostRAScheduling: get the minimum optimization level needed
// to enable post-RA scheduling.
virtual CodeGenOpt::Level getOptLevelToEnablePostRAScheduler() const {
@@ -284,6 +291,14 @@
/// This is called after a .mir file was loaded.
virtual void mirFileLoaded(MachineFunction &MF) const;
+
+ /// True if the register allocator should use the allocation orders exactly as
+ /// written in the tablegen descriptions, false if it should allocate
+ /// the specified physical register later if is it callee-saved.
+ virtual bool ignoreCSRForAllocationOrder(const MachineFunction &MF,
+ unsigned PhysReg) const {
+ return false;
+ }
};
} // end namespace llvm
diff --git a/linux-x64/clang/include/llvm/CodeGen/ValueTypes.td b/linux-x64/clang/include/llvm/CodeGen/ValueTypes.td
index f82faf2..feea7e5 100644
--- a/linux-x64/clang/include/llvm/CodeGen/ValueTypes.td
+++ b/linux-x64/clang/include/llvm/CodeGen/ValueTypes.td
@@ -62,89 +62,105 @@
def v64i16 : ValueType<1024,39>; // 64 x i16 vector value
def v128i16: ValueType<2048,40>; //128 x i16 vector value
-def v1i32 : ValueType<32 , 41>; // 1 x i32 vector value
-def v2i32 : ValueType<64 , 42>; // 2 x i32 vector value
-def v4i32 : ValueType<128, 43>; // 4 x i32 vector value
-def v8i32 : ValueType<256, 44>; // 8 x i32 vector value
-def v16i32 : ValueType<512, 45>; // 16 x i32 vector value
-def v32i32 : ValueType<1024,46>; // 32 x i32 vector value
-def v64i32 : ValueType<2048,47>; // 64 x i32 vector value
+def v1i32 : ValueType<32 , 41>; // 1 x i32 vector value
+def v2i32 : ValueType<64 , 42>; // 2 x i32 vector value
+def v3i32 : ValueType<96 , 43>; // 3 x i32 vector value
+def v4i32 : ValueType<128, 44>; // 4 x i32 vector value
+def v5i32 : ValueType<160, 45>; // 5 x i32 vector value
+def v8i32 : ValueType<256, 46>; // 8 x i32 vector value
+def v16i32 : ValueType<512, 47>; // 16 x i32 vector value
+def v32i32 : ValueType<1024,48>; // 32 x i32 vector value
+def v64i32 : ValueType<2048,49>; // 64 x i32 vector value
+def v128i32 : ValueType<4096,50>; // 128 x i32 vector value
+def v256i32 : ValueType<8182,51>; // 256 x i32 vector value
+def v512i32 : ValueType<16384,52>; // 512 x i32 vector value
+def v1024i32 : ValueType<32768,53>; // 1024 x i32 vector value
+def v2048i32 : ValueType<65536,54>; // 2048 x i32 vector value
-def v1i64 : ValueType<64 , 48>; // 1 x i64 vector value
-def v2i64 : ValueType<128, 49>; // 2 x i64 vector value
-def v4i64 : ValueType<256, 50>; // 4 x i64 vector value
-def v8i64 : ValueType<512, 51>; // 8 x i64 vector value
-def v16i64 : ValueType<1024,52>; // 16 x i64 vector value
-def v32i64 : ValueType<2048,53>; // 32 x i64 vector value
+def v1i64 : ValueType<64 , 55>; // 1 x i64 vector value
+def v2i64 : ValueType<128, 56>; // 2 x i64 vector value
+def v4i64 : ValueType<256, 57>; // 4 x i64 vector value
+def v8i64 : ValueType<512, 58>; // 8 x i64 vector value
+def v16i64 : ValueType<1024,59>; // 16 x i64 vector value
+def v32i64 : ValueType<2048,60>; // 32 x i64 vector value
-def v1i128 : ValueType<128, 54>; // 1 x i128 vector value
+def v1i128 : ValueType<128, 61>; // 1 x i128 vector value
-def nxv1i1 : ValueType<1, 55>; // n x 1 x i1 vector value
-def nxv2i1 : ValueType<2, 56>; // n x 2 x i1 vector value
-def nxv4i1 : ValueType<4, 57>; // n x 4 x i1 vector value
-def nxv8i1 : ValueType<8, 58>; // n x 8 x i1 vector value
-def nxv16i1 : ValueType<16, 59>; // n x 16 x i1 vector value
-def nxv32i1 : ValueType<32, 60>; // n x 32 x i1 vector value
+def nxv1i1 : ValueType<1, 62>; // n x 1 x i1 vector value
+def nxv2i1 : ValueType<2, 63>; // n x 2 x i1 vector value
+def nxv4i1 : ValueType<4, 64>; // n x 4 x i1 vector value
+def nxv8i1 : ValueType<8, 65>; // n x 8 x i1 vector value
+def nxv16i1 : ValueType<16, 66>; // n x 16 x i1 vector value
+def nxv32i1 : ValueType<32, 67>; // n x 32 x i1 vector value
-def nxv1i8 : ValueType<8, 61>; // n x 1 x i8 vector value
-def nxv2i8 : ValueType<16, 62>; // n x 2 x i8 vector value
-def nxv4i8 : ValueType<32, 63>; // n x 4 x i8 vector value
-def nxv8i8 : ValueType<64, 64>; // n x 8 x i8 vector value
-def nxv16i8 : ValueType<128, 65>; // n x 16 x i8 vector value
-def nxv32i8 : ValueType<256, 66>; // n x 32 x i8 vector value
+def nxv1i8 : ValueType<8, 68>; // n x 1 x i8 vector value
+def nxv2i8 : ValueType<16, 69>; // n x 2 x i8 vector value
+def nxv4i8 : ValueType<32, 70>; // n x 4 x i8 vector value
+def nxv8i8 : ValueType<64, 71>; // n x 8 x i8 vector value
+def nxv16i8 : ValueType<128, 72>; // n x 16 x i8 vector value
+def nxv32i8 : ValueType<256, 73>; // n x 32 x i8 vector value
-def nxv1i16 : ValueType<16, 67>; // n x 1 x i16 vector value
-def nxv2i16 : ValueType<32, 68>; // n x 2 x i16 vector value
-def nxv4i16 : ValueType<64, 69>; // n x 4 x i16 vector value
-def nxv8i16 : ValueType<128, 70>; // n x 8 x i16 vector value
-def nxv16i16: ValueType<256, 71>; // n x 16 x i16 vector value
-def nxv32i16: ValueType<512, 72>; // n x 32 x i16 vector value
+def nxv1i16 : ValueType<16, 74>; // n x 1 x i16 vector value
+def nxv2i16 : ValueType<32, 75>; // n x 2 x i16 vector value
+def nxv4i16 : ValueType<64, 76>; // n x 4 x i16 vector value
+def nxv8i16 : ValueType<128, 77>; // n x 8 x i16 vector value
+def nxv16i16: ValueType<256, 78>; // n x 16 x i16 vector value
+def nxv32i16: ValueType<512, 79>; // n x 32 x i16 vector value
-def nxv1i32 : ValueType<32, 73>; // n x 1 x i32 vector value
-def nxv2i32 : ValueType<64, 74>; // n x 2 x i32 vector value
-def nxv4i32 : ValueType<128, 75>; // n x 4 x i32 vector value
-def nxv8i32 : ValueType<256, 76>; // n x 8 x i32 vector value
-def nxv16i32: ValueType<512, 77>; // n x 16 x i32 vector value
-def nxv32i32: ValueType<1024,78>; // n x 32 x i32 vector value
+def nxv1i32 : ValueType<32, 80>; // n x 1 x i32 vector value
+def nxv2i32 : ValueType<64, 81>; // n x 2 x i32 vector value
+def nxv4i32 : ValueType<128, 82>; // n x 4 x i32 vector value
+def nxv8i32 : ValueType<256, 83>; // n x 8 x i32 vector value
+def nxv16i32: ValueType<512, 84>; // n x 16 x i32 vector value
+def nxv32i32: ValueType<1024,85>; // n x 32 x i32 vector value
-def nxv1i64 : ValueType<64, 79>; // n x 1 x i64 vector value
-def nxv2i64 : ValueType<128, 80>; // n x 2 x i64 vector value
-def nxv4i64 : ValueType<256, 81>; // n x 4 x i64 vector value
-def nxv8i64 : ValueType<512, 82>; // n x 8 x i64 vector value
-def nxv16i64: ValueType<1024,83>; // n x 16 x i64 vector value
-def nxv32i64: ValueType<2048,84>; // n x 32 x i64 vector value
+def nxv1i64 : ValueType<64, 86>; // n x 1 x i64 vector value
+def nxv2i64 : ValueType<128, 87>; // n x 2 x i64 vector value
+def nxv4i64 : ValueType<256, 88>; // n x 4 x i64 vector value
+def nxv8i64 : ValueType<512, 89>; // n x 8 x i64 vector value
+def nxv16i64: ValueType<1024,90>; // n x 16 x i64 vector value
+def nxv32i64: ValueType<2048,91>; // n x 32 x i64 vector value
-def v2f16 : ValueType<32 , 85>; // 2 x f16 vector value
-def v4f16 : ValueType<64 , 86>; // 4 x f16 vector value
-def v8f16 : ValueType<128, 87>; // 8 x f16 vector value
-def v1f32 : ValueType<32 , 88>; // 1 x f32 vector value
-def v2f32 : ValueType<64 , 89>; // 2 x f32 vector value
-def v4f32 : ValueType<128, 90>; // 4 x f32 vector value
-def v8f32 : ValueType<256, 91>; // 8 x f32 vector value
-def v16f32 : ValueType<512, 92>; // 16 x f32 vector value
-def v1f64 : ValueType<64, 93>; // 1 x f64 vector value
-def v2f64 : ValueType<128, 94>; // 2 x f64 vector value
-def v4f64 : ValueType<256, 95>; // 4 x f64 vector value
-def v8f64 : ValueType<512, 96>; // 8 x f64 vector value
+def v2f16 : ValueType<32 , 92>; // 2 x f16 vector value
+def v4f16 : ValueType<64 , 93>; // 4 x f16 vector value
+def v8f16 : ValueType<128, 94>; // 8 x f16 vector value
+def v1f32 : ValueType<32 , 95>; // 1 x f32 vector value
+def v2f32 : ValueType<64 , 96>; // 2 x f32 vector value
+def v3f32 : ValueType<96 , 97>; // 3 x f32 vector value
+def v4f32 : ValueType<128, 98>; // 4 x f32 vector value
+def v5f32 : ValueType<160, 99>; // 5 x f32 vector value
+def v8f32 : ValueType<256, 100>; // 8 x f32 vector value
+def v16f32 : ValueType<512, 101>; // 16 x f32 vector value
+def v32f32 : ValueType<1024, 102>; // 32 x f32 vector value
+def v64f32 : ValueType<2048, 103>; // 64 x f32 vector value
+def v128f32 : ValueType<4096, 104>; // 128 x f32 vector value
+def v256f32 : ValueType<8182, 105>; // 256 x f32 vector value
+def v512f32 : ValueType<16384, 106>; // 512 x f32 vector value
+def v1024f32 : ValueType<32768, 107>; // 1024 x f32 vector value
+def v2048f32 : ValueType<65536, 108>; // 2048 x f32 vector value
+def v1f64 : ValueType<64, 109>; // 1 x f64 vector value
+def v2f64 : ValueType<128, 110>; // 2 x f64 vector value
+def v4f64 : ValueType<256, 111>; // 4 x f64 vector value
+def v8f64 : ValueType<512, 112>; // 8 x f64 vector value
-def nxv2f16 : ValueType<32 , 97>; // n x 2 x f16 vector value
-def nxv4f16 : ValueType<64 , 98>; // n x 4 x f16 vector value
-def nxv8f16 : ValueType<128, 99>; // n x 8 x f16 vector value
-def nxv1f32 : ValueType<32 , 100>; // n x 1 x f32 vector value
-def nxv2f32 : ValueType<64 , 101>; // n x 2 x f32 vector value
-def nxv4f32 : ValueType<128, 102>; // n x 4 x f32 vector value
-def nxv8f32 : ValueType<256, 103>; // n x 8 x f32 vector value
-def nxv16f32 : ValueType<512, 104>; // n x 16 x f32 vector value
-def nxv1f64 : ValueType<64, 105>; // n x 1 x f64 vector value
-def nxv2f64 : ValueType<128, 106>; // n x 2 x f64 vector value
-def nxv4f64 : ValueType<256, 107>; // n x 4 x f64 vector value
-def nxv8f64 : ValueType<512, 108>; // n x 8 x f64 vector value
+def nxv2f16 : ValueType<32 , 113>; // n x 2 x f16 vector value
+def nxv4f16 : ValueType<64 , 114>; // n x 4 x f16 vector value
+def nxv8f16 : ValueType<128, 115>; // n x 8 x f16 vector value
+def nxv1f32 : ValueType<32 , 116>; // n x 1 x f32 vector value
+def nxv2f32 : ValueType<64 , 117>; // n x 2 x f32 vector value
+def nxv4f32 : ValueType<128, 118>; // n x 4 x f32 vector value
+def nxv8f32 : ValueType<256, 119>; // n x 8 x f32 vector value
+def nxv16f32 : ValueType<512, 120>; // n x 16 x f32 vector value
+def nxv1f64 : ValueType<64, 121>; // n x 1 x f64 vector value
+def nxv2f64 : ValueType<128, 122>; // n x 2 x f64 vector value
+def nxv4f64 : ValueType<256, 123>; // n x 4 x f64 vector value
+def nxv8f64 : ValueType<512, 124>; // n x 8 x f64 vector value
-def x86mmx : ValueType<64 , 109>; // X86 MMX value
-def FlagVT : ValueType<0 , 110>; // Pre-RA sched glue
-def isVoid : ValueType<0 , 111>; // Produces no value
-def untyped: ValueType<8 , 112>; // Produces an untyped value
-def ExceptRef: ValueType<0, 113>; // WebAssembly's except_ref type
+def x86mmx : ValueType<64 , 125>; // X86 MMX value
+def FlagVT : ValueType<0 , 126>; // Pre-RA sched glue
+def isVoid : ValueType<0 , 127>; // Produces no value
+def untyped: ValueType<8 , 128>; // Produces an untyped value
+def ExceptRef: ValueType<0, 129>; // WebAssembly's except_ref type
def token : ValueType<0 , 248>; // TokenTy
def MetadataVT: ValueType<0, 249>; // Metadata
@@ -166,3 +182,14 @@
// Pseudo valuetype to represent "any type of any size".
def Any : ValueType<0 , 255>;
+
+/// This class is for targets that want to use pointer types in patterns
+/// with the GlobalISelEmitter. Targets must define their own pointer
+/// derived from this class. The scalar argument should be an
+/// integer type with the same bit size as the ponter.
+/// e.g. def p0 : PtrValueType <i64, 0>;
+
+class PtrValueType <ValueType scalar, int addrspace> :
+ ValueType<scalar.Size, scalar.Value> {
+ int AddrSpace = addrspace;
+}
diff --git a/linux-x64/clang/include/llvm/CodeGen/VirtRegMap.h b/linux-x64/clang/include/llvm/CodeGen/VirtRegMap.h
index 7bdecbe..7a64d67 100644
--- a/linux-x64/clang/include/llvm/CodeGen/VirtRegMap.h
+++ b/linux-x64/clang/include/llvm/CodeGen/VirtRegMap.h
@@ -97,8 +97,8 @@
/// returns the physical register mapped to the specified
/// virtual register
- unsigned getPhys(unsigned virtReg) const {
- assert(TargetRegisterInfo::isVirtualRegister(virtReg));
+ Register getPhys(Register virtReg) const {
+ assert(virtReg.isVirtual());
return Virt2PhysMap[virtReg];
}
diff --git a/linux-x64/clang/include/llvm/CodeGen/WasmEHFuncInfo.h b/linux-x64/clang/include/llvm/CodeGen/WasmEHFuncInfo.h
index aaca847..887a146 100644
--- a/linux-x64/clang/include/llvm/CodeGen/WasmEHFuncInfo.h
+++ b/linux-x64/clang/include/llvm/CodeGen/WasmEHFuncInfo.h
@@ -28,10 +28,6 @@
// When there is an entry <A, B>, if an exception is not caught by A, it
// should next unwind to the EH pad B.
DenseMap<BBOrMBB, BBOrMBB> EHPadUnwindMap;
- // For entry <A, B>, A is a BB with an instruction that may throw
- // (invoke/cleanupret in LLVM IR, call/rethrow in the backend) and B is an EH
- // pad that A unwinds to.
- DenseMap<BBOrMBB, BBOrMBB> ThrowUnwindMap;
// Helper functions
const BasicBlock *getEHPadUnwindDest(const BasicBlock *BB) const {
@@ -40,18 +36,9 @@
void setEHPadUnwindDest(const BasicBlock *BB, const BasicBlock *Dest) {
EHPadUnwindMap[BB] = Dest;
}
- const BasicBlock *getThrowUnwindDest(BasicBlock *BB) const {
- return ThrowUnwindMap.lookup(BB).get<const BasicBlock *>();
- }
- void setThrowUnwindDest(const BasicBlock *BB, const BasicBlock *Dest) {
- ThrowUnwindMap[BB] = Dest;
- }
bool hasEHPadUnwindDest(const BasicBlock *BB) const {
return EHPadUnwindMap.count(BB);
}
- bool hasThrowUnwindDest(const BasicBlock *BB) const {
- return ThrowUnwindMap.count(BB);
- }
MachineBasicBlock *getEHPadUnwindDest(MachineBasicBlock *MBB) const {
return EHPadUnwindMap.lookup(MBB).get<MachineBasicBlock *>();
@@ -59,18 +46,9 @@
void setEHPadUnwindDest(MachineBasicBlock *MBB, MachineBasicBlock *Dest) {
EHPadUnwindMap[MBB] = Dest;
}
- MachineBasicBlock *getThrowUnwindDest(MachineBasicBlock *MBB) const {
- return ThrowUnwindMap.lookup(MBB).get<MachineBasicBlock *>();
- }
- void setThrowUnwindDest(MachineBasicBlock *MBB, MachineBasicBlock *Dest) {
- ThrowUnwindMap[MBB] = Dest;
- }
bool hasEHPadUnwindDest(MachineBasicBlock *MBB) const {
return EHPadUnwindMap.count(MBB);
}
- bool hasThrowUnwindDest(MachineBasicBlock *MBB) const {
- return ThrowUnwindMap.count(MBB);
- }
};
// Analyze the IR in the given function to build WasmEHFuncInfo.