Update prebuilt Clang to r416183b from Android.
https://android.googlesource.com/platform/prebuilts/clang/host/
linux-x86/+/06a71ddac05c22edb2d10b590e1769b3f8619bef
clang 12.0.5 (based on r416183b) from build 7284624.
Change-Id: I277a316abcf47307562d8b748b84870f31a72866
Signed-off-by: Olivier Deprez <olivier.deprez@arm.com>
diff --git a/linux-x64/clang/include/llvm/CodeGen/TargetLowering.h b/linux-x64/clang/include/llvm/CodeGen/TargetLowering.h
index d5cca60..305107c 100644
--- a/linux-x64/clang/include/llvm/CodeGen/TargetLowering.h
+++ b/linux-x64/clang/include/llvm/CodeGen/TargetLowering.h
@@ -28,7 +28,6 @@
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
-#include "llvm/Analysis/LegacyDivergenceAnalysis.h"
#include "llvm/CodeGen/DAGCombine.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/RuntimeLibcalls.h"
@@ -37,7 +36,6 @@
#include "llvm/CodeGen/TargetCallingConv.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/Attributes.h"
-#include "llvm/IR/CallSite.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
@@ -47,12 +45,11 @@
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Type.h"
-#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/Support/Alignment.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MachineValueType.h"
-#include "llvm/Target/TargetMachine.h"
#include <algorithm>
#include <cassert>
#include <climits>
@@ -72,8 +69,10 @@
class FastISel;
class FunctionLoweringInfo;
class GlobalValue;
+class GISelKnownBits;
class IntrinsicInst;
struct KnownBits;
+class LegacyDivergenceAnalysis;
class LLVMContext;
class MachineBasicBlock;
class MachineFunction;
@@ -84,9 +83,12 @@
class MCContext;
class MCExpr;
class Module;
-class TargetRegisterClass;
+class ProfileSummaryInfo;
class TargetLibraryInfo;
+class TargetMachine;
+class TargetRegisterClass;
class TargetRegisterInfo;
+class TargetTransformInfo;
class Value;
namespace Sched {
@@ -102,6 +104,85 @@
} // end namespace Sched
+// MemOp models a memory operation, either memset or memcpy/memmove.
+struct MemOp {
+private:
+ // Shared
+ uint64_t Size;
+ bool DstAlignCanChange; // true if destination alignment can satisfy any
+ // constraint.
+ Align DstAlign; // Specified alignment of the memory operation.
+
+ bool AllowOverlap;
+ // memset only
+ bool IsMemset; // If setthis memory operation is a memset.
+ bool ZeroMemset; // If set clears out memory with zeros.
+ // memcpy only
+ bool MemcpyStrSrc; // Indicates whether the memcpy source is an in-register
+ // constant so it does not need to be loaded.
+ Align SrcAlign; // Inferred alignment of the source or default value if the
+ // memory operation does not need to load the value.
+public:
+ static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign,
+ Align SrcAlign, bool IsVolatile,
+ bool MemcpyStrSrc = false) {
+ MemOp Op;
+ Op.Size = Size;
+ Op.DstAlignCanChange = DstAlignCanChange;
+ Op.DstAlign = DstAlign;
+ Op.AllowOverlap = !IsVolatile;
+ Op.IsMemset = false;
+ Op.ZeroMemset = false;
+ Op.MemcpyStrSrc = MemcpyStrSrc;
+ Op.SrcAlign = SrcAlign;
+ return Op;
+ }
+
+ static MemOp Set(uint64_t Size, bool DstAlignCanChange, Align DstAlign,
+ bool IsZeroMemset, bool IsVolatile) {
+ MemOp Op;
+ Op.Size = Size;
+ Op.DstAlignCanChange = DstAlignCanChange;
+ Op.DstAlign = DstAlign;
+ Op.AllowOverlap = !IsVolatile;
+ Op.IsMemset = true;
+ Op.ZeroMemset = IsZeroMemset;
+ Op.MemcpyStrSrc = false;
+ return Op;
+ }
+
+ uint64_t size() const { return Size; }
+ Align getDstAlign() const {
+ assert(!DstAlignCanChange);
+ return DstAlign;
+ }
+ bool isFixedDstAlign() const { return !DstAlignCanChange; }
+ bool allowOverlap() const { return AllowOverlap; }
+ bool isMemset() const { return IsMemset; }
+ bool isMemcpy() const { return !IsMemset; }
+ bool isMemcpyWithFixedDstAlign() const {
+ return isMemcpy() && !DstAlignCanChange;
+ }
+ bool isZeroMemset() const { return isMemset() && ZeroMemset; }
+ bool isMemcpyStrSrc() const {
+ assert(isMemcpy() && "Must be a memcpy");
+ return MemcpyStrSrc;
+ }
+ Align getSrcAlign() const {
+ assert(isMemcpy() && "Must be a memcpy");
+ return SrcAlign;
+ }
+ bool isSrcAligned(Align AlignCheck) const {
+ return isMemset() || llvm::isAligned(AlignCheck, SrcAlign.value());
+ }
+ bool isDstAligned(Align AlignCheck) const {
+ return DstAlignCanChange || llvm::isAligned(AlignCheck, DstAlign.value());
+ }
+ bool isAligned(Align AlignCheck) const {
+ return isSrcAligned(AlignCheck) && isDstAligned(AlignCheck);
+ }
+};
+
/// This base class for TargetLowering contains the SelectionDAG-independent
/// parts that can be used from the rest of CodeGen.
class TargetLoweringBase {
@@ -122,13 +203,20 @@
TypeLegal, // The target natively supports this type.
TypePromoteInteger, // Replace this integer with a larger one.
TypeExpandInteger, // Split this integer into two of half the size.
- TypeSoftenFloat, // Convert this float to a same size integer type,
- // if an operation is not supported in target HW.
+ TypeSoftenFloat, // Convert this float to a same size integer type.
TypeExpandFloat, // Split this float into two of half the size.
TypeScalarizeVector, // Replace this one-element vector with its element.
TypeSplitVector, // Split this vector into two of half the size.
TypeWidenVector, // This vector should be widened into a larger vector.
- TypePromoteFloat // Replace this float with a larger one.
+ TypePromoteFloat, // Replace this float with a larger one.
+ TypeSoftPromoteHalf, // Soften half to i16 and use float to do arithmetic.
+ TypeScalarizeScalableVector, // This action is explicitly left unimplemented.
+ // While it is theoretically possible to
+ // legalize operations on scalable types with a
+ // loop that handles the vscale * #lanes of the
+ // vector, this is non-trivial at SelectionDAG
+ // level and these types are better to be
+ // widened or promoted.
};
/// LegalizeKind holds the legalization kind that needs to happen to EVT
@@ -172,6 +260,13 @@
// or custom.
};
+ /// Enum that specifies when a float negation is beneficial.
+ enum class NegatibleCost {
+ Cheaper = 0, // Negated expression is cheaper.
+ Neutral = 1, // Negated expression has the same cost.
+ Expensive = 2 // Negated expression is more expensive.
+ };
+
class ArgListEntry {
public:
Value *Val = nullptr;
@@ -183,23 +278,24 @@
bool IsSRet : 1;
bool IsNest : 1;
bool IsByVal : 1;
+ bool IsByRef : 1;
bool IsInAlloca : 1;
+ bool IsPreallocated : 1;
bool IsReturned : 1;
bool IsSwiftSelf : 1;
bool IsSwiftError : 1;
- uint16_t Alignment = 0;
+ bool IsCFGuardTarget : 1;
+ MaybeAlign Alignment = None;
Type *ByValType = nullptr;
+ Type *PreallocatedType = nullptr;
ArgListEntry()
: IsSExt(false), IsZExt(false), IsInReg(false), IsSRet(false),
- IsNest(false), IsByVal(false), IsInAlloca(false), IsReturned(false),
- IsSwiftSelf(false), IsSwiftError(false) {}
+ IsNest(false), IsByVal(false), IsByRef(false), IsInAlloca(false),
+ IsPreallocated(false), IsReturned(false), IsSwiftSelf(false),
+ IsSwiftError(false), IsCFGuardTarget(false) {}
void setAttributes(const CallBase *Call, unsigned ArgIdx);
-
- void setAttributes(ImmutableCallSite *CS, unsigned ArgIdx) {
- return setAttributes(cast<CallBase>(CS->getInstruction()), ArgIdx);
- }
};
using ArgListTy = std::vector<ArgListEntry>;
@@ -221,12 +317,16 @@
llvm_unreachable("Invalid content kind");
}
- /// NOTE: The TargetMachine owns TLOF.
explicit TargetLoweringBase(const TargetMachine &TM);
TargetLoweringBase(const TargetLoweringBase &) = delete;
TargetLoweringBase &operator=(const TargetLoweringBase &) = delete;
virtual ~TargetLoweringBase() = default;
+ /// Return true if the target support strict float operation
+ bool isStrictFPEnabled() const {
+ return IsStrictFPEnabled;
+ }
+
protected:
/// Initialize all of the actions to default values.
void initActions();
@@ -256,6 +356,12 @@
return getPointerTy(DL, DL.getAllocaAddrSpace());
}
+ /// Return the type for code pointers, which is determined by the program
+ /// address space specified through the data layout.
+ MVT getProgramPointerTy(const DataLayout &DL) const {
+ return getPointerTy(DL, DL.getProgramAddressSpace());
+ }
+
/// Return the type for operands of fence.
/// TODO: Let fence operands be of i32 type and remove this.
virtual MVT getFenceOperandTy(const DataLayout &DL) const {
@@ -269,6 +375,13 @@
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL,
bool LegalTypes = true) const;
+ /// Return the preferred type to use for a shift opcode, given the shifted
+ /// amount type is \p ShiftValueTy.
+ LLVM_READONLY
+ virtual LLT getPreferredShiftAmountTy(LLT ShiftValueTy) const {
+ return ShiftValueTy;
+ }
+
/// Returns the type to be used for the index operand of:
/// ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
/// ISD::INSERT_SUBVECTOR, and ISD::EXTRACT_SUBVECTOR
@@ -276,6 +389,20 @@
return getPointerTy(DL);
}
+ /// This callback is used to inspect load/store instructions and add
+ /// target-specific MachineMemOperand flags to them. The default
+ /// implementation does nothing.
+ virtual MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const {
+ return MachineMemOperand::MONone;
+ }
+
+ MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI,
+ const DataLayout &DL) const;
+ MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI,
+ const DataLayout &DL) const;
+ MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI,
+ const DataLayout &DL) const;
+
virtual bool isSelectSupported(SelectSupportKind /*kind*/) const {
return true;
}
@@ -284,7 +411,7 @@
/// a constant pool load whose address depends on the select condition. The
/// parameter may be used to differentiate a select with FP compare from
/// integer compare.
- virtual bool reduceSelectOfFPConstantLoads(bool IsFPSetCC) const {
+ virtual bool reduceSelectOfFPConstantLoads(EVT CmpOpVT) const {
return true;
}
@@ -300,7 +427,7 @@
virtual TargetLoweringBase::LegalizeTypeAction
getPreferredVectorAction(MVT VT) const {
// The default action for one element vectors is to scalarize
- if (VT.getVectorNumElements() == 1)
+ if (VT.getVectorElementCount().isScalar())
return TypeScalarizeVector;
// The default action for an odd-width vector is to widen.
if (!VT.isPow2VectorType())
@@ -309,6 +436,12 @@
return TypePromoteInteger;
}
+ // Return true if the half type should be passed around as i16, but promoted
+ // to float around arithmetic. The default behavior is to pass around as
+ // float and convert around loads/stores/bitcasts and other places where
+ // the size matters.
+ virtual bool softPromoteHalfType() const { return false; }
+
// There are two general methods for expanding a BUILD_VECTOR node:
// 1. Use SCALAR_TO_VECTOR on the defined scalar values and then shuffle
// them together.
@@ -391,6 +524,10 @@
return PredictableSelectIsExpensive;
}
+ virtual bool fallBackToDAGISel(const Instruction &Inst) const {
+ return false;
+ }
+
/// If a branch or a select condition is skewed in one direction by more than
/// this factor, it is very likely to be predicted correctly.
virtual BranchProbability getPredictableBranchThreshold() const;
@@ -468,6 +605,16 @@
return false;
}
+ /// Return the maximum number of "x & (x - 1)" operations that can be done
+ /// instead of deferring to a custom CTPOP.
+ virtual unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const {
+ return 1;
+ }
+
+ /// Return true if instruction generated for equality comparison is folded
+ /// with instruction generated for signed comparison.
+ virtual bool isEqualityCmpFoldedWithSignedCmp() const { return true; }
+
/// Return true if it is safe to transform an integer-domain bitwise operation
/// into the equivalent floating-point operation. This should be set to true
/// if the target has IEEE-754-compliant fabs/fneg operations for the input
@@ -539,6 +686,12 @@
return hasAndNotCompare(X);
}
+ /// Return true if the target has a bit-test instruction:
+ /// (X & (1 << Y)) ==/!= 0
+ /// This knowledge can be used to prevent breaking the pattern,
+ /// or creating it if it could be recognized.
+ virtual bool hasBitTest(SDValue X, SDValue Y) const { return false; }
+
/// There are two ways to clear extreme bits (either low or high):
/// Mask: x & (-1 << y) (the instcombine canonical form)
/// Shifts: x >> y << y
@@ -571,6 +724,38 @@
return false;
}
+ /// Given the pattern
+ /// (X & (C l>>/<< Y)) ==/!= 0
+ /// return true if it should be transformed into:
+ /// ((X <</l>> Y) & C) ==/!= 0
+ /// WARNING: if 'X' is a constant, the fold may deadlock!
+ /// FIXME: we could avoid passing XC, but we can't use isConstOrConstSplat()
+ /// here because it can end up being not linked in.
+ virtual bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
+ SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
+ unsigned OldShiftOpcode, unsigned NewShiftOpcode,
+ SelectionDAG &DAG) const {
+ if (hasBitTest(X, Y)) {
+ // One interesting pattern that we'd want to form is 'bit test':
+ // ((1 << Y) & C) ==/!= 0
+ // But we also need to be careful not to try to reverse that fold.
+
+ // Is this '1 << Y' ?
+ if (OldShiftOpcode == ISD::SHL && CC->isOne())
+ return false; // Keep the 'bit test' pattern.
+
+ // Will it be '1 << Y' after the transform ?
+ if (XC && NewShiftOpcode == ISD::SHL && XC->isOne())
+ return true; // Do form the 'bit test' pattern.
+ }
+
+ // If 'X' is a constant, and we transform, then we will immediately
+ // try to undo the fold, thus causing endless combine loop.
+ // So by default, let's assume everyone prefers the fold
+ // iff 'X' is not a constant.
+ return !XC;
+ }
+
/// These two forms are equivalent:
/// sub %y, (xor %x, -1)
/// add (add %x, 1), %y
@@ -798,9 +983,9 @@
PointerUnion<const Value *, const PseudoSourceValue *> ptrVal;
int offset = 0; // offset off of ptrVal
- unsigned size = 0; // the size of the memory location
+ uint64_t size = 0; // the size of the memory location
// (taken from memVT if zero)
- unsigned align = 1; // alignment
+ MaybeAlign align = Align(1); // alignment
MachineMemOperand::Flags flags = MachineMemOperand::MONone;
IntrinsicInfo() = default;
@@ -884,6 +1069,11 @@
case ISD::SMULFIX:
case ISD::SMULFIXSAT:
case ISD::UMULFIX:
+ case ISD::UMULFIXSAT:
+ case ISD::SDIVFIX:
+ case ISD::SDIVFIXSAT:
+ case ISD::UDIVFIX:
+ case ISD::UDIVFIXSAT:
Supported = isSupportedFixedPointOperation(Op, VT, Scale);
break;
}
@@ -891,52 +1081,31 @@
return Supported ? Action : Expand;
}
+ // If Op is a strict floating-point operation, return the result
+ // of getOperationAction for the equivalent non-strict operation.
LegalizeAction getStrictFPOperationAction(unsigned Op, EVT VT) const {
unsigned EqOpc;
switch (Op) {
default: llvm_unreachable("Unexpected FP pseudo-opcode");
- case ISD::STRICT_FADD: EqOpc = ISD::FADD; break;
- case ISD::STRICT_FSUB: EqOpc = ISD::FSUB; break;
- case ISD::STRICT_FMUL: EqOpc = ISD::FMUL; break;
- case ISD::STRICT_FDIV: EqOpc = ISD::FDIV; break;
- case ISD::STRICT_FREM: EqOpc = ISD::FREM; break;
- case ISD::STRICT_FSQRT: EqOpc = ISD::FSQRT; break;
- case ISD::STRICT_FPOW: EqOpc = ISD::FPOW; break;
- case ISD::STRICT_FPOWI: EqOpc = ISD::FPOWI; break;
- case ISD::STRICT_FMA: EqOpc = ISD::FMA; break;
- case ISD::STRICT_FSIN: EqOpc = ISD::FSIN; break;
- case ISD::STRICT_FCOS: EqOpc = ISD::FCOS; break;
- case ISD::STRICT_FEXP: EqOpc = ISD::FEXP; break;
- case ISD::STRICT_FEXP2: EqOpc = ISD::FEXP2; break;
- case ISD::STRICT_FLOG: EqOpc = ISD::FLOG; break;
- case ISD::STRICT_FLOG10: EqOpc = ISD::FLOG10; break;
- case ISD::STRICT_FLOG2: EqOpc = ISD::FLOG2; break;
- case ISD::STRICT_FRINT: EqOpc = ISD::FRINT; break;
- case ISD::STRICT_FNEARBYINT: EqOpc = ISD::FNEARBYINT; break;
- case ISD::STRICT_FMAXNUM: EqOpc = ISD::FMAXNUM; break;
- case ISD::STRICT_FMINNUM: EqOpc = ISD::FMINNUM; break;
- case ISD::STRICT_FCEIL: EqOpc = ISD::FCEIL; break;
- case ISD::STRICT_FFLOOR: EqOpc = ISD::FFLOOR; break;
- case ISD::STRICT_FROUND: EqOpc = ISD::FROUND; break;
- case ISD::STRICT_FTRUNC: EqOpc = ISD::FTRUNC; break;
- case ISD::STRICT_FP_ROUND: EqOpc = ISD::FP_ROUND; break;
- case ISD::STRICT_FP_EXTEND: EqOpc = ISD::FP_EXTEND; break;
+#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
+ case ISD::STRICT_##DAGN: EqOpc = ISD::DAGN; break;
+#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
+ case ISD::STRICT_##DAGN: EqOpc = ISD::SETCC; break;
+#include "llvm/IR/ConstrainedOps.def"
}
- auto Action = getOperationAction(EqOpc, VT);
-
- // We don't currently handle Custom or Promote for strict FP pseudo-ops.
- // For now, we just expand for those cases.
- if (Action != Legal)
- Action = Expand;
-
- return Action;
+ return getOperationAction(EqOpc, VT);
}
/// Return true if the specified operation is legal on this target or can be
/// made legal with custom lowering. This is used to help guide high-level
- /// lowering decisions.
- bool isOperationLegalOrCustom(unsigned Op, EVT VT) const {
+ /// lowering decisions. LegalOnly is an optional convenience for code paths
+ /// traversed pre and post legalisation.
+ bool isOperationLegalOrCustom(unsigned Op, EVT VT,
+ bool LegalOnly = false) const {
+ if (LegalOnly)
+ return isOperationLegal(Op, VT);
+
return (VT == MVT::Other || isTypeLegal(VT)) &&
(getOperationAction(Op, VT) == Legal ||
getOperationAction(Op, VT) == Custom);
@@ -944,8 +1113,13 @@
/// Return true if the specified operation is legal on this target or can be
/// made legal using promotion. This is used to help guide high-level lowering
- /// decisions.
- bool isOperationLegalOrPromote(unsigned Op, EVT VT) const {
+ /// decisions. LegalOnly is an optional convenience for code paths traversed
+ /// pre and post legalisation.
+ bool isOperationLegalOrPromote(unsigned Op, EVT VT,
+ bool LegalOnly = false) const {
+ if (LegalOnly)
+ return isOperationLegal(Op, VT);
+
return (VT == MVT::Other || isTypeLegal(VT)) &&
(getOperationAction(Op, VT) == Legal ||
getOperationAction(Op, VT) == Promote);
@@ -953,8 +1127,13 @@
/// Return true if the specified operation is legal on this target or can be
/// made legal with custom lowering or using promotion. This is used to help
- /// guide high-level lowering decisions.
- bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT) const {
+ /// guide high-level lowering decisions. LegalOnly is an optional convenience
+ /// for code paths traversed pre and post legalisation.
+ bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT,
+ bool LegalOnly = false) const {
+ if (LegalOnly)
+ return isOperationLegal(Op, VT);
+
return (VT == MVT::Other || isTypeLegal(VT)) &&
(getOperationAction(Op, VT) == Legal ||
getOperationAction(Op, VT) == Custom ||
@@ -988,24 +1167,8 @@
/// Return true if lowering to a jump table is suitable for a set of case
/// clusters which may contain \p NumCases cases, \p Range range of values.
virtual bool isSuitableForJumpTable(const SwitchInst *SI, uint64_t NumCases,
- uint64_t Range) const {
- // FIXME: This function check the maximum table size and density, but the
- // minimum size is not checked. It would be nice if the minimum size is
- // also combined within this function. Currently, the minimum size check is
- // performed in findJumpTable() in SelectionDAGBuiler and
- // getEstimatedNumberOfCaseClusters() in BasicTTIImpl.
- const bool OptForSize = SI->getParent()->getParent()->hasOptSize();
- const unsigned MinDensity = getMinimumJumpTableDensity(OptForSize);
- const unsigned MaxJumpTableSize = getMaximumJumpTableSize();
-
- // Check whether the number of cases is small enough and
- // the range is dense enough for a jump table.
- if ((OptForSize || Range <= MaxJumpTableSize) &&
- (NumCases * 100 >= Range * MinDensity)) {
- return true;
- }
- return false;
- }
+ uint64_t Range, ProfileSummaryInfo *PSI,
+ BlockFrequencyInfo *BFI) const;
/// Return true if lowering to a bit test is suitable for a set of case
/// clusters which contains \p NumDests unique destinations, \p Low and
@@ -1102,12 +1265,8 @@
/// Return how the indexed load should be treated: either it is legal, needs
/// to be promoted to a larger size, needs to be expanded to some other code
/// sequence, or the target has a custom expander for it.
- LegalizeAction
- getIndexedLoadAction(unsigned IdxMode, MVT VT) const {
- assert(IdxMode < ISD::LAST_INDEXED_MODE && VT.isValid() &&
- "Table isn't big enough!");
- unsigned Ty = (unsigned)VT.SimpleTy;
- return (LegalizeAction)((IndexedModeActions[Ty][IdxMode] & 0xf0) >> 4);
+ LegalizeAction getIndexedLoadAction(unsigned IdxMode, MVT VT) const {
+ return getIndexedModeAction(IdxMode, VT, IMAB_Load);
}
/// Return true if the specified indexed load is legal on this target.
@@ -1120,12 +1279,8 @@
/// Return how the indexed store should be treated: either it is legal, needs
/// to be promoted to a larger size, needs to be expanded to some other code
/// sequence, or the target has a custom expander for it.
- LegalizeAction
- getIndexedStoreAction(unsigned IdxMode, MVT VT) const {
- assert(IdxMode < ISD::LAST_INDEXED_MODE && VT.isValid() &&
- "Table isn't big enough!");
- unsigned Ty = (unsigned)VT.SimpleTy;
- return (LegalizeAction)(IndexedModeActions[Ty][IdxMode] & 0x0f);
+ LegalizeAction getIndexedStoreAction(unsigned IdxMode, MVT VT) const {
+ return getIndexedModeAction(IdxMode, VT, IMAB_Store);
}
/// Return true if the specified indexed load is legal on this target.
@@ -1135,6 +1290,38 @@
getIndexedStoreAction(IdxMode, VT.getSimpleVT()) == Custom);
}
+ /// Return how the indexed load should be treated: either it is legal, needs
+ /// to be promoted to a larger size, needs to be expanded to some other code
+ /// sequence, or the target has a custom expander for it.
+ LegalizeAction getIndexedMaskedLoadAction(unsigned IdxMode, MVT VT) const {
+ return getIndexedModeAction(IdxMode, VT, IMAB_MaskedLoad);
+ }
+
+ /// Return true if the specified indexed load is legal on this target.
+ bool isIndexedMaskedLoadLegal(unsigned IdxMode, EVT VT) const {
+ return VT.isSimple() &&
+ (getIndexedMaskedLoadAction(IdxMode, VT.getSimpleVT()) == Legal ||
+ getIndexedMaskedLoadAction(IdxMode, VT.getSimpleVT()) == Custom);
+ }
+
+ /// Return how the indexed store should be treated: either it is legal, needs
+ /// to be promoted to a larger size, needs to be expanded to some other code
+ /// sequence, or the target has a custom expander for it.
+ LegalizeAction getIndexedMaskedStoreAction(unsigned IdxMode, MVT VT) const {
+ return getIndexedModeAction(IdxMode, VT, IMAB_MaskedStore);
+ }
+
+ /// Return true if the specified indexed load is legal on this target.
+ bool isIndexedMaskedStoreLegal(unsigned IdxMode, EVT VT) const {
+ return VT.isSimple() &&
+ (getIndexedMaskedStoreAction(IdxMode, VT.getSimpleVT()) == Legal ||
+ getIndexedMaskedStoreAction(IdxMode, VT.getSimpleVT()) == Custom);
+ }
+
+ // Returns true if VT is a legal index type for masked gathers/scatters
+ // on this target
+ virtual bool shouldRemoveExtendFromGSIndex(EVT VT) const { return false; }
+
/// Return how the condition code should be treated: either it is legal, needs
/// to be expanded to some other code sequence, or the target has a custom
/// expander for it.
@@ -1206,7 +1393,7 @@
EltTy = PointerTy.getTypeForEVT(Ty->getContext());
}
return EVT::getVectorVT(Ty->getContext(), EVT::getEVT(EltTy, false),
- VTy->getNumElements());
+ VTy->getElementCount());
}
return EVT::getEVT(Ty, AllowUnknown);
@@ -1224,7 +1411,7 @@
Elm = PointerTy.getTypeForEVT(Ty->getContext());
}
return EVT::getVectorVT(Ty->getContext(), EVT::getEVT(Elm, false),
- VTy->getNumElements());
+ VTy->getElementCount());
}
return getValueType(DL, Ty, AllowUnknown);
@@ -1316,9 +1503,9 @@
/// Certain targets have context senstive alignment requirements, where one
/// type has the alignment requirement of another type.
- virtual unsigned getABIAlignmentForCallingConv(Type *ArgTy,
- DataLayout DL) const {
- return DL.getABITypeAlignment(ArgTy);
+ virtual Align getABIAlignmentForCallingConv(Type *ArgTy,
+ DataLayout DL) const {
+ return DL.getABITypeAlign(ArgTy);
}
/// If true, then instruction selection should seek to shrink the FP constant
@@ -1426,13 +1613,40 @@
return false;
}
+ /// LLT handling variant.
+ virtual bool allowsMisalignedMemoryAccesses(
+ LLT, unsigned AddrSpace = 0, Align Alignment = Align(1),
+ MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
+ bool * /*Fast*/ = nullptr) const {
+ return false;
+ }
+
+ /// This function returns true if the memory access is aligned or if the
+ /// target allows this specific unaligned memory access. If the access is
+ /// allowed, the optional final parameter returns if the access is also fast
+ /// (as defined by the target).
+ bool allowsMemoryAccessForAlignment(
+ LLVMContext &Context, const DataLayout &DL, EVT VT,
+ unsigned AddrSpace = 0, Align Alignment = Align(1),
+ MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
+ bool *Fast = nullptr) const;
+
+ /// Return true if the memory access of this type is aligned or if the target
+ /// allows this specific unaligned access for the given MachineMemOperand.
+ /// If the access is allowed, the optional final parameter returns if the
+ /// access is also fast (as defined by the target).
+ bool allowsMemoryAccessForAlignment(LLVMContext &Context,
+ const DataLayout &DL, EVT VT,
+ const MachineMemOperand &MMO,
+ bool *Fast = nullptr) const;
+
/// Return true if the target supports a memory access of this type for the
/// given address space and alignment. If the access is allowed, the optional
/// final parameter returns if the access is also fast (as defined by the
/// target).
- bool
+ virtual bool
allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT,
- unsigned AddrSpace = 0, unsigned Alignment = 1,
+ unsigned AddrSpace = 0, Align Alignment = Align(1),
MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
bool *Fast = nullptr) const;
@@ -1446,23 +1660,21 @@
/// Returns the target specific optimal type for load and store operations as
/// a result of memset, memcpy, and memmove lowering.
- ///
- /// If DstAlign is zero that means it's safe to destination alignment can
- /// satisfy any constraint. Similarly if SrcAlign is zero it means there isn't
- /// a need to check it against alignment requirement, probably because the
- /// source does not need to be loaded. If 'IsMemset' is true, that means it's
- /// expanding a memset. If 'ZeroMemset' is true, that means it's a memset of
- /// zero. 'MemcpyStrSrc' indicates whether the memcpy source is constant so it
- /// does not need to be loaded. It returns EVT::Other if the type should be
- /// determined using generic target-independent logic.
+ /// It returns EVT::Other if the type should be determined using generic
+ /// target-independent logic.
virtual EVT
- getOptimalMemOpType(uint64_t /*Size*/, unsigned /*DstAlign*/,
- unsigned /*SrcAlign*/, bool /*IsMemset*/,
- bool /*ZeroMemset*/, bool /*MemcpyStrSrc*/,
+ getOptimalMemOpType(const MemOp &Op,
const AttributeList & /*FuncAttributes*/) const {
return MVT::Other;
}
+ /// LLT returning variant.
+ virtual LLT
+ getOptimalMemOpLLT(const MemOp &Op,
+ const AttributeList & /*FuncAttributes*/) const {
+ return LLT();
+ }
+
/// Returns true if it's safe to use load / store of the specified type to
/// expand memcpy / memset inline.
///
@@ -1472,16 +1684,6 @@
/// have to be legal as the hook is used before type legalization.
virtual bool isSafeMemOpType(MVT /*VT*/) const { return true; }
- /// Determine if we should use _setjmp or setjmp to implement llvm.setjmp.
- bool usesUnderscoreSetJmp() const {
- return UseUnderscoreSetJmp;
- }
-
- /// Determine if we should use _longjmp or longjmp to implement llvm.longjmp.
- bool usesUnderscoreLongJmp() const {
- return UseUnderscoreLongJmp;
- }
-
/// Return lower limit for number of blocks in a jump table.
virtual unsigned getMinimumJumpTableEntries() const;
@@ -1492,65 +1694,45 @@
/// Zero if no limit.
unsigned getMaximumJumpTableSize() const;
- virtual bool isJumpTableRelative() const {
- return TM.isPositionIndependent();
- }
+ virtual bool isJumpTableRelative() const;
/// If a physical register, this specifies the register that
/// llvm.savestack/llvm.restorestack should save and restore.
- unsigned getStackPointerRegisterToSaveRestore() const {
+ Register getStackPointerRegisterToSaveRestore() const {
return StackPointerRegisterToSaveRestore;
}
/// If a physical register, this returns the register that receives the
/// exception address on entry to an EH pad.
- virtual unsigned
+ virtual Register
getExceptionPointerRegister(const Constant *PersonalityFn) const {
- // 0 is guaranteed to be the NoRegister value on all targets
- return 0;
+ return Register();
}
/// If a physical register, this returns the register that receives the
/// exception typeid on entry to a landing pad.
- virtual unsigned
+ virtual Register
getExceptionSelectorRegister(const Constant *PersonalityFn) const {
- // 0 is guaranteed to be the NoRegister value on all targets
- return 0;
+ return Register();
}
virtual bool needsFixedCatchObjects() const {
report_fatal_error("Funclet EH is not implemented for this target");
}
- /// Returns the target's jmp_buf size in bytes (if never set, the default is
- /// 200)
- unsigned getJumpBufSize() const {
- return JumpBufSize;
- }
-
- /// Returns the target's jmp_buf alignment in bytes (if never set, the default
- /// is 0)
- unsigned getJumpBufAlignment() const {
- return JumpBufAlignment;
- }
-
/// Return the minimum stack alignment of an argument.
- unsigned getMinStackArgumentAlignment() const {
+ Align getMinStackArgumentAlignment() const {
return MinStackArgumentAlignment;
}
/// Return the minimum function alignment.
- unsigned getMinFunctionAlignment() const {
- return MinFunctionAlignment;
- }
+ Align getMinFunctionAlignment() const { return MinFunctionAlignment; }
/// Return the preferred function alignment.
- unsigned getPrefFunctionAlignment() const {
- return PrefFunctionAlignment;
- }
+ Align getPrefFunctionAlignment() const { return PrefFunctionAlignment; }
/// Return the preferred loop alignment.
- virtual unsigned getPrefLoopAlignment(MachineLoop *ML = nullptr) const {
+ virtual Align getPrefLoopAlignment(MachineLoop *ML = nullptr) const {
return PrefLoopAlignment;
}
@@ -1597,21 +1779,18 @@
/// Returns the name of the symbol used to emit stack probes or the empty
/// string if not applicable.
+ virtual bool hasStackProbeSymbol(MachineFunction &MF) const { return false; }
+
+ virtual bool hasInlineStackProbe(MachineFunction &MF) const { return false; }
+
virtual StringRef getStackProbeSymbolName(MachineFunction &MF) const {
return "";
}
- /// Returns true if a cast between SrcAS and DestAS is a noop.
- virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const {
- return false;
- }
-
/// Returns true if a cast from SrcAS to DestAS is "cheap", such that e.g. we
/// are happy to sink it into basic blocks. A cast may be free, but not
/// necessarily a no-op. e.g. a free truncate from a 64-bit to 32-bit pointer.
- virtual bool isFreeAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const {
- return isNoopAddrSpaceCast(SrcAS, DestAS);
- }
+ virtual bool isFreeAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const;
/// Return true if the pointer arguments to CI should be aligned by aligning
/// the object whose address is being passed. If so then MinSize is set to the
@@ -1772,6 +1951,11 @@
return IsSigned;
}
+ /// Returns true if arguments should be extended in lib calls.
+ virtual bool shouldExtendTypeInLibCall(EVT Type) const {
+ return true;
+ }
+
/// Returns how the given (atomic) load should be expanded by the
/// IR-level AtomicExpand pass.
virtual AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const {
@@ -1814,6 +1998,18 @@
return ISD::ZERO_EXTEND;
}
+ /// Returns how the platform's atomic compare and swap expects its comparison
+ /// value to be extended (ZERO_EXTEND, SIGN_EXTEND, or ANY_EXTEND). This is
+ /// separate from getExtendForAtomicOps, which is concerned with the
+ /// sign-extension of the instruction's output, whereas here we are concerned
+ /// with the sign-extension of the input. For targets with compare-and-swap
+ /// instructions (or sub-word comparisons in their LL/SC loop expansions),
+ /// the input can be ANY_EXTEND, but the output will still have a specific
+ /// extension.
+ virtual ISD::NodeType getExtendForAtomicCmpSwapArg() const {
+ return ISD::ANY_EXTEND;
+ }
+
/// @}
/// Returns true if we should normalize
@@ -1848,7 +2044,8 @@
/// This may be true if the target does not directly support the
/// multiplication operation for the specified type or the sequence of simpler
/// ops is faster than the multiply.
- virtual bool decomposeMulByConstant(EVT VT, SDValue C) const {
+ virtual bool decomposeMulByConstant(LLVMContext &Context,
+ EVT VT, SDValue C) const {
return false;
}
@@ -1892,18 +2089,6 @@
SchedPreferenceInfo = Pref;
}
- /// Indicate whether this target prefers to use _setjmp to implement
- /// llvm.setjmp or the version without _. Defaults to false.
- void setUseUnderscoreSetJmp(bool Val) {
- UseUnderscoreSetJmp = Val;
- }
-
- /// Indicate whether this target prefers to use _longjmp to implement
- /// llvm.longjmp or the version without _. Defaults to false.
- void setUseUnderscoreLongJmp(bool Val) {
- UseUnderscoreLongJmp = Val;
- }
-
/// Indicate the minimum number of blocks to generate jump tables.
void setMinimumJumpTableEntries(unsigned Val);
@@ -1913,7 +2098,7 @@
/// If set to a physical register, this specifies the register that
/// llvm.savestack/llvm.restorestack should save and restore.
- void setStackPointerRegisterToSaveRestore(unsigned R) {
+ void setStackPointerRegisterToSaveRestore(Register R) {
StackPointerRegisterToSaveRestore = R;
}
@@ -1995,13 +2180,8 @@
///
/// NOTE: All indexed mode loads are initialized to Expand in
/// TargetLowering.cpp
- void setIndexedLoadAction(unsigned IdxMode, MVT VT,
- LegalizeAction Action) {
- assert(VT.isValid() && IdxMode < ISD::LAST_INDEXED_MODE &&
- (unsigned)Action < 0xf && "Table isn't big enough!");
- // Load action are kept in the upper half.
- IndexedModeActions[(unsigned)VT.SimpleTy][IdxMode] &= ~0xf0;
- IndexedModeActions[(unsigned)VT.SimpleTy][IdxMode] |= ((uint8_t)Action) <<4;
+ void setIndexedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action) {
+ setIndexedModeAction(IdxMode, VT, IMAB_Load, Action);
}
/// Indicate that the specified indexed store does or does not work with the
@@ -2009,13 +2189,28 @@
///
/// NOTE: All indexed mode stores are initialized to Expand in
/// TargetLowering.cpp
- void setIndexedStoreAction(unsigned IdxMode, MVT VT,
- LegalizeAction Action) {
- assert(VT.isValid() && IdxMode < ISD::LAST_INDEXED_MODE &&
- (unsigned)Action < 0xf && "Table isn't big enough!");
- // Store action are kept in the lower half.
- IndexedModeActions[(unsigned)VT.SimpleTy][IdxMode] &= ~0x0f;
- IndexedModeActions[(unsigned)VT.SimpleTy][IdxMode] |= ((uint8_t)Action);
+ void setIndexedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action) {
+ setIndexedModeAction(IdxMode, VT, IMAB_Store, Action);
+ }
+
+ /// Indicate that the specified indexed masked load does or does not work with
+ /// the specified type and indicate what to do about it.
+ ///
+ /// NOTE: All indexed mode masked loads are initialized to Expand in
+ /// TargetLowering.cpp
+ void setIndexedMaskedLoadAction(unsigned IdxMode, MVT VT,
+ LegalizeAction Action) {
+ setIndexedModeAction(IdxMode, VT, IMAB_MaskedLoad, Action);
+ }
+
+ /// Indicate that the specified indexed masked store does or does not work
+ /// with the specified type and indicate what to do about it.
+ ///
+ /// NOTE: All indexed mode masked stores are initialized to Expand in
+ /// TargetLowering.cpp
+ void setIndexedMaskedStoreAction(unsigned IdxMode, MVT VT,
+ LegalizeAction Action) {
+ setIndexedModeAction(IdxMode, VT, IMAB_MaskedStore, Action);
}
/// Indicate that the specified condition code is or isn't supported on the
@@ -2056,40 +2251,25 @@
TargetDAGCombineArray[NT >> 3] |= 1 << (NT&7);
}
- /// Set the target's required jmp_buf buffer size (in bytes); default is 200
- void setJumpBufSize(unsigned Size) {
- JumpBufSize = Size;
- }
-
- /// Set the target's required jmp_buf buffer alignment (in bytes); default is
- /// 0
- void setJumpBufAlignment(unsigned Align) {
- JumpBufAlignment = Align;
- }
-
- /// Set the target's minimum function alignment (in log2(bytes))
- void setMinFunctionAlignment(unsigned Align) {
- MinFunctionAlignment = Align;
+ /// Set the target's minimum function alignment.
+ void setMinFunctionAlignment(Align Alignment) {
+ MinFunctionAlignment = Alignment;
}
/// Set the target's preferred function alignment. This should be set if
- /// there is a performance benefit to higher-than-minimum alignment (in
- /// log2(bytes))
- void setPrefFunctionAlignment(unsigned Align) {
- PrefFunctionAlignment = Align;
+ /// there is a performance benefit to higher-than-minimum alignment
+ void setPrefFunctionAlignment(Align Alignment) {
+ PrefFunctionAlignment = Alignment;
}
- /// Set the target's preferred loop alignment. Default alignment is zero, it
- /// means the target does not care about loop alignment. The alignment is
- /// specified in log2(bytes). The target may also override
- /// getPrefLoopAlignment to provide per-loop values.
- void setPrefLoopAlignment(unsigned Align) {
- PrefLoopAlignment = Align;
- }
+ /// Set the target's preferred loop alignment. Default alignment is one, it
+ /// means the target does not care about loop alignment. The target may also
+ /// override getPrefLoopAlignment to provide per-loop values.
+ void setPrefLoopAlignment(Align Alignment) { PrefLoopAlignment = Alignment; }
- /// Set the minimum stack alignment of an argument (in log2(bytes)).
- void setMinStackArgumentAlignment(unsigned Align) {
- MinStackArgumentAlignment = Align;
+ /// Set the minimum stack alignment of an argument.
+ void setMinStackArgumentAlignment(Align Alignment) {
+ MinStackArgumentAlignment = Alignment;
}
/// Set the maximum atomic operation size supported by the
@@ -2193,13 +2373,31 @@
}
/// Return true if it's significantly cheaper to shift a vector by a uniform
- /// scalar than by an amount which will vary across each lane. On x86, for
- /// example, there is a "psllw" instruction for the former case, but no simple
- /// instruction for a general "a << b" operation on vectors.
+ /// scalar than by an amount which will vary across each lane. On x86 before
+ /// AVX2 for example, there is a "psllw" instruction for the former case, but
+ /// no simple instruction for a general "a << b" operation on vectors.
+ /// This should also apply to lowering for vector funnel shifts (rotates).
virtual bool isVectorShiftByScalarCheap(Type *Ty) const {
return false;
}
+ /// Given a shuffle vector SVI representing a vector splat, return a new
+ /// scalar type of size equal to SVI's scalar type if the new type is more
+ /// profitable. Returns nullptr otherwise. For example under MVE float splats
+ /// are converted to integer to prevent the need to move from SPR to GPR
+ /// registers.
+ virtual Type* shouldConvertSplatType(ShuffleVectorInst* SVI) const {
+ return nullptr;
+ }
+
+ /// Given a set in interconnected phis of type 'From' that are loaded/stored
+ /// or bitcast to type 'To', return true if the set should be converted to
+ /// 'To'.
+ virtual bool shouldConvertPhiType(Type *From, Type *To) const {
+ return (From->isIntegerTy() || From->isFloatingPointTy()) &&
+ (To->isIntegerTy() || To->isFloatingPointTy());
+ }
+
/// Returns true if the opcode is a commutative binary operation.
virtual bool isCommutativeBinOp(unsigned Opcode) const {
// FIXME: This should get its info from the td file.
@@ -2396,7 +2594,7 @@
/// this information should not be provided because it will generate more
/// loads.
virtual bool hasPairedLoad(EVT /*LoadedType*/,
- unsigned & /*RequiredAlignment*/) const {
+ Align & /*RequiredAlignment*/) const {
return false;
}
@@ -2451,7 +2649,8 @@
/// Return true if an fpext operation input to an \p Opcode operation is free
/// (for instance, because half-precision floating-point numbers are
/// implicitly extended to float-precision) for an FMA instruction.
- virtual bool isFPExtFoldable(unsigned Opcode, EVT DestVT, EVT SrcVT) const {
+ virtual bool isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode,
+ EVT DestVT, EVT SrcVT) const {
assert(DestVT.isFloatingPoint() && SrcVT.isFloatingPoint() &&
"invalid fpext types");
return isFPExtFree(DestVT, SrcVT);
@@ -2483,10 +2682,26 @@
/// not legal, but should return true if those types will eventually legalize
/// to types that support FMAs. After legalization, it will only be called on
/// types that support FMAs (via Legal or Custom actions)
- virtual bool isFMAFasterThanFMulAndFAdd(EVT) const {
+ virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
+ EVT) const {
return false;
}
+ /// IR version
+ virtual bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *) const {
+ return false;
+ }
+
+ /// Returns true if be combined with to form an ISD::FMAD. \p N may be an
+ /// ISD::FADD, ISD::FSUB, or an ISD::FMUL which will be distributed into an
+ /// fadd/fsub.
+ virtual bool isFMADLegal(const SelectionDAG &DAG, const SDNode *N) const {
+ assert((N->getOpcode() == ISD::FADD || N->getOpcode() == ISD::FSUB ||
+ N->getOpcode() == ISD::FMUL) &&
+ "unexpected node in FMAD forming combine");
+ return isOperationLegal(ISD::FMAD, N->getValueType(0));
+ }
+
/// Return true if it's profitable to narrow operations of type VT1 to
/// VT2. e.g. on x86, it's profitable to narrow from i32 to i8 but not from
/// i32 to i16.
@@ -2531,17 +2746,21 @@
/// node operation. Targets may want to override this independently of whether
/// the operation is legal/custom for the given type because it may obscure
/// matching of other patterns.
- virtual bool shouldFormOverflowOp(unsigned Opcode, EVT VT) const {
+ virtual bool shouldFormOverflowOp(unsigned Opcode, EVT VT,
+ bool MathUsed) const {
// TODO: The default logic is inherited from code in CodeGenPrepare.
// The opcode should not make a difference by default?
if (Opcode != ISD::UADDO)
return false;
// Allow the transform as long as we have an integer type that is not
- // obviously illegal and unsupported.
+ // obviously illegal and unsupported and if the math result is used
+ // besides the overflow check. On some targets (e.g. SPARC), it is
+ // not profitable to form on overflow op if the math result has no
+ // concrete users.
if (VT.isVector())
return false;
- return VT.isSimple() || !isOperationExpand(Opcode, VT);
+ return MathUsed && (VT.isSimple() || !isOperationExpand(Opcode, VT));
}
// Return true if it is profitable to use a scalar input to a BUILD_VECTOR
@@ -2555,6 +2774,12 @@
// same blocks of its users.
virtual bool shouldConsiderGEPOffsetSplit() const { return false; }
+ /// Return true if creating a shift of the type by the given
+ /// amount is not profitable.
+ virtual bool shouldAvoidTransformToShift(EVT VT, unsigned Amount) const {
+ return false;
+ }
+
//===--------------------------------------------------------------------===//
// Runtime Library hooks
//
@@ -2597,6 +2822,13 @@
/// The default implementation just freezes the set of reserved registers.
virtual void finalizeLowering(MachineFunction &MF) const;
+ //===----------------------------------------------------------------------===//
+ // GlobalISel Hooks
+ //===----------------------------------------------------------------------===//
+ /// Check whether or not \p MI needs to be moved close to its uses.
+ virtual bool shouldLocalize(const MachineInstr &MI, const TargetTransformInfo *TTI) const;
+
+
private:
const TargetMachine &TM;
@@ -2624,16 +2856,6 @@
/// predication.
bool JumpIsExpensive;
- /// This target prefers to use _setjmp to implement llvm.setjmp.
- ///
- /// Defaults to false.
- bool UseUnderscoreSetJmp;
-
- /// This target prefers to use _longjmp to implement llvm.longjmp.
- ///
- /// Defaults to false.
- bool UseUnderscoreLongJmp;
-
/// Information about the contents of the high-bits in boolean values held in
/// a type wider than i1. See getBooleanContents.
BooleanContent BooleanContents;
@@ -2650,25 +2872,19 @@
/// register usage.
Sched::Preference SchedPreferenceInfo;
- /// The size, in bytes, of the target's jmp_buf buffers
- unsigned JumpBufSize;
-
- /// The alignment, in bytes, of the target's jmp_buf buffers
- unsigned JumpBufAlignment;
-
/// The minimum alignment that any argument on the stack needs to have.
- unsigned MinStackArgumentAlignment;
+ Align MinStackArgumentAlignment;
/// The minimum function alignment (used when optimizing for size, and to
/// prevent explicitly provided alignment from leading to incorrect code).
- unsigned MinFunctionAlignment;
+ Align MinFunctionAlignment;
/// The preferred function alignment (used when alignment unspecified and
/// optimizing for speed).
- unsigned PrefFunctionAlignment;
+ Align PrefFunctionAlignment;
- /// The preferred loop alignment.
- unsigned PrefLoopAlignment;
+ /// The preferred loop alignment (in log2 bot in bytes).
+ Align PrefLoopAlignment;
/// Size in bits of the maximum atomics size the backend supports.
/// Accesses larger than this will be expanded by AtomicExpandPass.
@@ -2683,12 +2899,12 @@
/// If set to a physical register, this specifies the register that
/// llvm.savestack/llvm.restorestack should save and restore.
- unsigned StackPointerRegisterToSaveRestore;
+ Register StackPointerRegisterToSaveRestore;
/// This indicates the default register class to use for each ValueType the
/// target supports natively.
const TargetRegisterClass *RegClassForVT[MVT::LAST_VALUETYPE];
- unsigned char NumRegistersForVT[MVT::LAST_VALUETYPE];
+ uint16_t NumRegistersForVT[MVT::LAST_VALUETYPE];
MVT RegisterTypeForVT[MVT::LAST_VALUETYPE];
/// This indicates the "representative" register class to use for each
@@ -2728,13 +2944,13 @@
/// truncating store of a specific value type and truncating type is legal.
LegalizeAction TruncStoreActions[MVT::LAST_VALUETYPE][MVT::LAST_VALUETYPE];
- /// For each indexed mode and each value type, keep a pair of LegalizeAction
+ /// For each indexed mode and each value type, keep a quad of LegalizeAction
/// that indicates how instruction selection should deal with the load /
- /// store.
+ /// store / maskedload / maskedstore.
///
/// The first dimension is the value_type for the reference. The second
/// dimension represents the various modes for load store.
- uint8_t IndexedModeActions[MVT::LAST_VALUETYPE][ISD::LAST_INDEXED_MODE];
+ uint16_t IndexedModeActions[MVT::LAST_VALUETYPE][ISD::LAST_INDEXED_MODE];
/// For each condition code (ISD::CondCode) keep a LegalizeAction that
/// indicates how instruction selection should deal with the condition code.
@@ -2744,7 +2960,6 @@
/// up the MVT::LAST_VALUETYPE value to the next multiple of 8.
uint32_t CondCodeActions[ISD::SETCC_INVALID][(MVT::LAST_VALUETYPE + 7) / 8];
-protected:
ValueTypeActionImpl ValueTypeActions;
private:
@@ -2778,6 +2993,32 @@
/// Set default libcall names and calling conventions.
void InitLibcalls(const Triple &TT);
+ /// The bits of IndexedModeActions used to store the legalisation actions
+ /// We store the data as | ML | MS | L | S | each taking 4 bits.
+ enum IndexedModeActionsBits {
+ IMAB_Store = 0,
+ IMAB_Load = 4,
+ IMAB_MaskedStore = 8,
+ IMAB_MaskedLoad = 12
+ };
+
+ void setIndexedModeAction(unsigned IdxMode, MVT VT, unsigned Shift,
+ LegalizeAction Action) {
+ assert(VT.isValid() && IdxMode < ISD::LAST_INDEXED_MODE &&
+ (unsigned)Action < 0xf && "Table isn't big enough!");
+ unsigned Ty = (unsigned)VT.SimpleTy;
+ IndexedModeActions[Ty][IdxMode] &= ~(0xf << Shift);
+ IndexedModeActions[Ty][IdxMode] |= ((uint16_t)Action) << Shift;
+ }
+
+ LegalizeAction getIndexedModeAction(unsigned IdxMode, MVT VT,
+ unsigned Shift) const {
+ assert(IdxMode < ISD::LAST_INDEXED_MODE && VT.isValid() &&
+ "Table isn't big enough!");
+ unsigned Ty = (unsigned)VT.SimpleTy;
+ return (LegalizeAction)((IndexedModeActions[Ty][IdxMode] >> Shift) & 0xf);
+ }
+
protected:
/// Return true if the extension represented by \p I is free.
/// \pre \p I is a sign, zero, or fp extension and
@@ -2790,7 +3031,7 @@
/// expected to be merged.
unsigned GatherAllAliasesMaxDepth;
- /// Specify maximum number of store instructions per memset call.
+ /// \brief Specify maximum number of store instructions per memset call.
///
/// When lowering \@llvm.memset this field specifies the maximum number of
/// store operations that may be substituted for the call to memset. Targets
@@ -2801,12 +3042,10 @@
/// with 16-bit alignment would result in four 2-byte stores and one 1-byte
/// store. This only applies to setting a constant array of a constant size.
unsigned MaxStoresPerMemset;
-
- /// Maximum number of stores operations that may be substituted for the call
- /// to memset, used for functions with OptSize attribute.
+ /// Likewise for functions with the OptSize attribute.
unsigned MaxStoresPerMemsetOptSize;
- /// Specify maximum bytes of store instructions per memcpy call.
+ /// \brief Specify maximum number of store instructions per memcpy call.
///
/// When lowering \@llvm.memcpy this field specifies the maximum number of
/// store operations that may be substituted for a call to memcpy. Targets
@@ -2818,8 +3057,8 @@
/// and one 1-byte store. This only applies to copying a constant array of
/// constant size.
unsigned MaxStoresPerMemcpy;
-
-
+ /// Likewise for functions with the OptSize attribute.
+ unsigned MaxStoresPerMemcpyOptSize;
/// \brief Specify max number of store instructions to glue in inlined memcpy.
///
/// When memcpy is inlined based on MaxStoresPerMemcpy, specify maximum number
@@ -2827,13 +3066,22 @@
// vectorization later on.
unsigned MaxGluedStoresPerMemcpy = 0;
- /// Maximum number of store operations that may be substituted for a call to
- /// memcpy, used for functions with OptSize attribute.
- unsigned MaxStoresPerMemcpyOptSize;
+ /// \brief Specify maximum number of load instructions per memcmp call.
+ ///
+ /// When lowering \@llvm.memcmp this field specifies the maximum number of
+ /// pairs of load operations that may be substituted for a call to memcmp.
+ /// Targets must set this value based on the cost threshold for that target.
+ /// Targets should assume that the memcmp will be done using as many of the
+ /// largest load operations first, followed by smaller ones, if necessary, per
+ /// alignment restrictions. For example, loading 7 bytes on a 32-bit machine
+ /// with 32-bit alignment would result in one 4-byte load, a one 2-byte load
+ /// and one 1-byte load. This only applies to copying a constant array of
+ /// constant size.
unsigned MaxLoadsPerMemcmp;
+ /// Likewise for functions with the OptSize attribute.
unsigned MaxLoadsPerMemcmpOptSize;
- /// Specify maximum bytes of store instructions per memmove call.
+ /// \brief Specify maximum number of store instructions per memmove call.
///
/// When lowering \@llvm.memmove this field specifies the maximum number of
/// store instructions that may be substituted for a call to memmove. Targets
@@ -2844,9 +3092,7 @@
/// with 8-bit alignment would result in nine 1-byte stores. This only
/// applies to copying a constant array of constant size.
unsigned MaxStoresPerMemmove;
-
- /// Maximum number of store instructions that may be substituted for a call to
- /// memmove, used for functions with OptSize attribute.
+ /// Likewise for functions with the OptSize attribute.
unsigned MaxStoresPerMemmoveOptSize;
/// Tells the code generator that select is more expensive than a branch if
@@ -2875,6 +3121,8 @@
/// details.
MachineBasicBlock *emitXRayTypedEvent(MachineInstr &MI,
MachineBasicBlock *MBB) const;
+
+ bool IsStrictFPEnabled;
};
/// This class defines information used to lower LLVM code to legal SelectionDAG
@@ -2885,11 +3133,11 @@
class TargetLowering : public TargetLoweringBase {
public:
struct DAGCombinerInfo;
+ struct MakeLibCallOptions;
TargetLowering(const TargetLowering &) = delete;
TargetLowering &operator=(const TargetLowering &) = delete;
- /// NOTE: The TargetMachine owns TLOF.
explicit TargetLowering(const TargetMachine &TM);
bool isPositionIndependent() const;
@@ -2925,6 +3173,14 @@
return false;
}
+ /// Returns true if the specified base+offset is a legal indexed addressing
+ /// mode for this target. \p MI is the load or store instruction that is being
+ /// considered for transformation.
+ virtual bool isIndexingLegal(MachineInstr &MI, Register Base, Register Offset,
+ bool IsPre, MachineRegisterInfo &MRI) const {
+ return false;
+ }
+
/// Return the entry encoding for a jump table in the current function. The
/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
virtual unsigned getJumpTableEncoding() const;
@@ -2955,14 +3211,22 @@
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS,
SDValue &NewRHS, ISD::CondCode &CCCode,
- const SDLoc &DL) const;
+ const SDLoc &DL, const SDValue OldLHS,
+ const SDValue OldRHS) const;
+
+ void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS,
+ SDValue &NewRHS, ISD::CondCode &CCCode,
+ const SDLoc &DL, const SDValue OldLHS,
+ const SDValue OldRHS, SDValue &Chain,
+ bool IsSignaling = false) const;
/// Returns a pair of (return value, chain).
/// It is an error to pass RTLIB::UNKNOWN_LIBCALL as \p LC.
- std::pair<SDValue, SDValue> makeLibCall(
- SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef<SDValue> Ops,
- bool isSigned, const SDLoc &dl, bool doesNotReturn = false,
- bool isReturnValueUsed = true, bool isPostTypeLegalization = false) const;
+ std::pair<SDValue, SDValue> makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC,
+ EVT RetVT, ArrayRef<SDValue> Ops,
+ MakeLibCallOptions CallOptions,
+ const SDLoc &dl,
+ SDValue Chain = SDValue()) const;
/// Check whether parameters to a call that are passed in callee saved
/// registers are the same as from the calling function. This needs to be
@@ -3004,27 +3268,28 @@
/// Return true if the number of memory ops is below the threshold (Limit).
/// It returns the types of the sequence of memory ops to perform
/// memset / memcpy by reference.
- bool findOptimalMemOpLowering(std::vector<EVT> &MemOps,
- unsigned Limit, uint64_t Size,
- unsigned DstAlign, unsigned SrcAlign,
- bool IsMemset,
- bool ZeroMemset,
- bool MemcpyStrSrc,
- bool AllowOverlap,
- unsigned DstAS, unsigned SrcAS,
+ bool findOptimalMemOpLowering(std::vector<EVT> &MemOps, unsigned Limit,
+ const MemOp &Op, unsigned DstAS, unsigned SrcAS,
const AttributeList &FuncAttributes) const;
/// Check to see if the specified operand of the specified instruction is a
/// constant integer. If so, check to see if there are any bits set in the
/// constant that are not demanded. If so, shrink the constant and return
/// true.
- bool ShrinkDemandedConstant(SDValue Op, const APInt &Demanded,
+ bool ShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits,
+ const APInt &DemandedElts,
+ TargetLoweringOpt &TLO) const;
+
+ /// Helper wrapper around ShrinkDemandedConstant, demanding all elements.
+ bool ShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits,
TargetLoweringOpt &TLO) const;
// Target hook to do target-specific const optimization, which is called by
// ShrinkDemandedConstant. This function should return true if the target
// doesn't want ShrinkDemandedConstant to further optimize the constant.
- virtual bool targetShrinkDemandedConstant(SDValue Op, const APInt &Demanded,
+ virtual bool targetShrinkDemandedConstant(SDValue Op,
+ const APInt &DemandedBits,
+ const APInt &DemandedElts,
TargetLoweringOpt &TLO) const {
return false;
}
@@ -3062,9 +3327,30 @@
/// Helper wrapper around SimplifyDemandedBits.
/// Adds Op back to the worklist upon success.
- bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedMask,
+ bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
DAGCombinerInfo &DCI) const;
+ /// More limited version of SimplifyDemandedBits that can be used to "look
+ /// through" ops that don't contribute to the DemandedBits/DemandedElts -
+ /// bitwise ops etc.
+ SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits,
+ const APInt &DemandedElts,
+ SelectionDAG &DAG,
+ unsigned Depth) const;
+
+ /// Helper wrapper around SimplifyMultipleUseDemandedBits, demanding all
+ /// elements.
+ SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits,
+ SelectionDAG &DAG,
+ unsigned Depth = 0) const;
+
+ /// Helper wrapper around SimplifyMultipleUseDemandedBits, demanding all
+ /// bits from only some vector elements.
+ SDValue SimplifyMultipleUseDemandedVectorElts(SDValue Op,
+ const APInt &DemandedElts,
+ SelectionDAG &DAG,
+ unsigned Depth = 0) const;
+
/// Look at Vector Op. At this point, we know that only the DemandedElts
/// elements of the result of Op are ever used downstream. If we can use
/// this information to simplify Op, create a new simplified DAG node and
@@ -3100,14 +3386,31 @@
const SelectionDAG &DAG,
unsigned Depth = 0) const;
+ /// Determine which of the bits specified in Mask are known to be either zero
+ /// or one and return them in the KnownZero/KnownOne bitsets. The DemandedElts
+ /// argument allows us to only collect the known bits that are shared by the
+ /// requested vector elements. This is for GISel.
+ virtual void computeKnownBitsForTargetInstr(GISelKnownBits &Analysis,
+ Register R, KnownBits &Known,
+ const APInt &DemandedElts,
+ const MachineRegisterInfo &MRI,
+ unsigned Depth = 0) const;
+
+ /// Determine the known alignment for the pointer value \p R. This is can
+ /// typically be inferred from the number of low known 0 bits. However, for a
+ /// pointer with a non-integral address space, the alignment value may be
+ /// independent from the known low bits.
+ virtual Align computeKnownAlignForTargetInstr(GISelKnownBits &Analysis,
+ Register R,
+ const MachineRegisterInfo &MRI,
+ unsigned Depth = 0) const;
+
/// Determine which of the bits of FrameIndex \p FIOp are known to be 0.
/// Default implementation computes low bits based on alignment
/// information. This should preserve known bits passed into it.
- virtual void computeKnownBitsForFrameIndex(const SDValue FIOp,
+ virtual void computeKnownBitsForFrameIndex(int FIOp,
KnownBits &Known,
- const APInt &DemandedElts,
- const SelectionDAG &DAG,
- unsigned Depth = 0) const;
+ const MachineFunction &MF) const;
/// This method can be implemented by targets that want to expose additional
/// information about sign bits to the DAG Combiner. The DemandedElts
@@ -3118,6 +3421,16 @@
const SelectionDAG &DAG,
unsigned Depth = 0) const;
+ /// This method can be implemented by targets that want to expose additional
+ /// information about sign bits to GlobalISel combiners. The DemandedElts
+ /// argument allows us to only collect the minimum sign bits that are shared
+ /// by the requested vector elements.
+ virtual unsigned computeNumSignBitsForTargetInstr(GISelKnownBits &Analysis,
+ Register R,
+ const APInt &DemandedElts,
+ const MachineRegisterInfo &MRI,
+ unsigned Depth = 0) const;
+
/// Attempt to simplify any target nodes based on the demanded vector
/// elements, returning true on success. Otherwise, analyze the expression and
/// return a mask of KnownUndef and KnownZero elements for the expression
@@ -3139,6 +3452,21 @@
TargetLoweringOpt &TLO,
unsigned Depth = 0) const;
+ /// More limited version of SimplifyDemandedBits that can be used to "look
+ /// through" ops that don't contribute to the DemandedBits/DemandedElts -
+ /// bitwise ops etc.
+ virtual SDValue SimplifyMultipleUseDemandedBitsForTargetNode(
+ SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
+ SelectionDAG &DAG, unsigned Depth) const;
+
+ /// Tries to build a legal vector shuffle using the provided parameters
+ /// or equivalent variations. The Mask argument maybe be modified as the
+ /// function tries different variations.
+ /// Returns an empty SDValue if the operation fails.
+ SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
+ SDValue N1, MutableArrayRef<int> Mask,
+ SelectionDAG &DAG) const;
+
/// This method returns the constant pool value that will be loaded by LD.
/// NOTE: You must check for implicit extensions of the constant by LD.
virtual const Constant *getTargetConstantFromLoad(LoadSDNode *LD) const;
@@ -3163,9 +3491,7 @@
bool isBeforeLegalize() const { return Level == BeforeLegalizeTypes; }
bool isBeforeLegalizeOps() const { return Level < AfterLegalizeVectorOps; }
- bool isAfterLegalizeDAG() const {
- return Level == AfterLegalizeDAG;
- }
+ bool isAfterLegalizeDAG() const { return Level >= AfterLegalizeDAG; }
CombineLevel getDAGCombineLevel() { return Level; }
bool isCalledByLegalizer() const { return CalledByLegalizer; }
@@ -3174,6 +3500,8 @@
SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true);
SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo = true);
+ bool recursivelyDeleteUnusedNodes(SDNode *N);
+
void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO);
};
@@ -3230,20 +3558,6 @@
return true;
}
- // Return true if it is profitable to combine a BUILD_VECTOR with a stride-pattern
- // to a shuffle and a truncate.
- // Example of such a combine:
- // v4i32 build_vector((extract_elt V, 1),
- // (extract_elt V, 3),
- // (extract_elt V, 5),
- // (extract_elt V, 7))
- // -->
- // v4i32 truncate (bitcast (shuffle<1,u,3,u,5,u,7,u> V, u) to v4i64)
- virtual bool isDesirableToCombineBuildVectorToShuffleTruncate(
- ArrayRef<int> ShuffleMask, EVT SrcVT, EVT TruncVT) const {
- return false;
- }
-
/// Return true if the target has native support for the specified value type
/// and it is 'desirable' to use the type for the given node type. e.g. On x86
/// i16 is legal, but undesirable since i16 instruction encodings are longer
@@ -3297,11 +3611,61 @@
llvm_unreachable("Not Implemented");
}
+ /// Return the newly negated expression if the cost is not expensive and
+ /// set the cost in \p Cost to indicate that if it is cheaper or neutral to
+ /// do the negation.
+ virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG,
+ bool LegalOps, bool OptForSize,
+ NegatibleCost &Cost,
+ unsigned Depth = 0) const;
+
+ /// This is the helper function to return the newly negated expression only
+ /// when the cost is cheaper.
+ SDValue getCheaperNegatedExpression(SDValue Op, SelectionDAG &DAG,
+ bool LegalOps, bool OptForSize,
+ unsigned Depth = 0) const {
+ NegatibleCost Cost = NegatibleCost::Expensive;
+ SDValue Neg =
+ getNegatedExpression(Op, DAG, LegalOps, OptForSize, Cost, Depth);
+ if (Neg && Cost == NegatibleCost::Cheaper)
+ return Neg;
+ // Remove the new created node to avoid the side effect to the DAG.
+ if (Neg && Neg.getNode()->use_empty())
+ DAG.RemoveDeadNode(Neg.getNode());
+ return SDValue();
+ }
+
+ /// This is the helper function to return the newly negated expression if
+ /// the cost is not expensive.
+ SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps,
+ bool OptForSize, unsigned Depth = 0) const {
+ NegatibleCost Cost = NegatibleCost::Expensive;
+ return getNegatedExpression(Op, DAG, LegalOps, OptForSize, Cost, Depth);
+ }
+
//===--------------------------------------------------------------------===//
// Lowering methods - These methods must be implemented by targets so that
// the SelectionDAGBuilder code knows how to lower these.
//
+ /// Target-specific splitting of values into parts that fit a register
+ /// storing a legal type
+ virtual bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL,
+ SDValue Val, SDValue *Parts,
+ unsigned NumParts, MVT PartVT,
+ Optional<CallingConv::ID> CC) const {
+ return false;
+ }
+
+ /// Target-specific combining of register parts into its original value
+ virtual SDValue
+ joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL,
+ const SDValue *Parts, unsigned NumParts,
+ MVT PartVT, EVT ValueVT,
+ Optional<CallingConv::ID> CC) const {
+ return SDValue();
+ }
+
/// This hook must be implemented to lower the incoming (formal) arguments,
/// described by the Ins array, into the specified DAG. The implementation
/// should fill in the InVals array with legal-type argument values, and
@@ -3328,6 +3692,8 @@
bool IsReturnValueUsed : 1;
bool IsConvergent : 1;
bool IsPatchPoint : 1;
+ bool IsPreallocated : 1;
+ bool NoMerge : 1;
// IsTailCall should be modified by implementations of
// TargetLowering::LowerCall that perform tail call conversions.
@@ -3342,7 +3708,7 @@
ArgListTy Args;
SelectionDAG &DAG;
SDLoc DL;
- ImmutableCallSite CS;
+ const CallBase *CB = nullptr;
SmallVector<ISD::OutputArg, 32> Outs;
SmallVector<SDValue, 32> OutVals;
SmallVector<ISD::InputArg, 32> Ins;
@@ -3351,7 +3717,8 @@
CallLoweringInfo(SelectionDAG &DAG)
: RetSExt(false), RetZExt(false), IsVarArg(false), IsInReg(false),
DoesNotReturn(false), IsReturnValueUsed(true), IsConvergent(false),
- IsPatchPoint(false), DAG(DAG) {}
+ IsPatchPoint(false), IsPreallocated(false), NoMerge(false),
+ DAG(DAG) {}
CallLoweringInfo &setDebugLoc(const SDLoc &dl) {
DL = dl;
@@ -3389,26 +3756,26 @@
CallLoweringInfo &setCallee(Type *ResultType, FunctionType *FTy,
SDValue Target, ArgListTy &&ArgsList,
- ImmutableCallSite Call) {
+ const CallBase &Call) {
RetTy = ResultType;
IsInReg = Call.hasRetAttr(Attribute::InReg);
DoesNotReturn =
Call.doesNotReturn() ||
- (!Call.isInvoke() &&
- isa<UnreachableInst>(Call.getInstruction()->getNextNode()));
+ (!isa<InvokeInst>(Call) && isa<UnreachableInst>(Call.getNextNode()));
IsVarArg = FTy->isVarArg();
- IsReturnValueUsed = !Call.getInstruction()->use_empty();
+ IsReturnValueUsed = !Call.use_empty();
RetSExt = Call.hasRetAttr(Attribute::SExt);
RetZExt = Call.hasRetAttr(Attribute::ZExt);
-
+ NoMerge = Call.hasFnAttr(Attribute::NoMerge);
+
Callee = Target;
CallConv = Call.getCallingConv();
NumFixedArgs = FTy->getNumParams();
Args = std::move(ArgsList);
- CS = Call;
+ CB = &Call;
return *this;
}
@@ -3458,6 +3825,11 @@
return *this;
}
+ CallLoweringInfo &setIsPreallocated(bool Value = true) {
+ IsPreallocated = Value;
+ return *this;
+ }
+
CallLoweringInfo &setIsPostTypeLegalization(bool Value=true) {
IsPostTypeLegalization = Value;
return *this;
@@ -3468,6 +3840,51 @@
}
};
+ /// This structure is used to pass arguments to makeLibCall function.
+ struct MakeLibCallOptions {
+ // By passing type list before soften to makeLibCall, the target hook
+ // shouldExtendTypeInLibCall can get the original type before soften.
+ ArrayRef<EVT> OpsVTBeforeSoften;
+ EVT RetVTBeforeSoften;
+ bool IsSExt : 1;
+ bool DoesNotReturn : 1;
+ bool IsReturnValueUsed : 1;
+ bool IsPostTypeLegalization : 1;
+ bool IsSoften : 1;
+
+ MakeLibCallOptions()
+ : IsSExt(false), DoesNotReturn(false), IsReturnValueUsed(true),
+ IsPostTypeLegalization(false), IsSoften(false) {}
+
+ MakeLibCallOptions &setSExt(bool Value = true) {
+ IsSExt = Value;
+ return *this;
+ }
+
+ MakeLibCallOptions &setNoReturn(bool Value = true) {
+ DoesNotReturn = Value;
+ return *this;
+ }
+
+ MakeLibCallOptions &setDiscardResult(bool Value = true) {
+ IsReturnValueUsed = !Value;
+ return *this;
+ }
+
+ MakeLibCallOptions &setIsPostTypeLegalization(bool Value = true) {
+ IsPostTypeLegalization = Value;
+ return *this;
+ }
+
+ MakeLibCallOptions &setTypeListBeforeSoften(ArrayRef<EVT> OpsVT, EVT RetVT,
+ bool Value = true) {
+ OpsVTBeforeSoften = OpsVT;
+ RetVTBeforeSoften = RetVT;
+ IsSoften = Value;
+ return *this;
+ }
+ };
+
/// This function lowers an abstract call to a function into an actual call.
/// This returns a pair of operands. The first element is the return value
/// for the function (if RetTy is not VoidTy). The second element is the
@@ -3486,7 +3903,7 @@
}
/// Target-specific cleanup for formal ByVal parameters.
- virtual void HandleByVal(CCState *, unsigned &, unsigned) const {}
+ virtual void HandleByVal(CCState *, unsigned &, Align) const {}
/// This hook should be implemented to check whether the return values
/// described by the Outs array can fit into the return registers. If false
@@ -3537,8 +3954,8 @@
/// Return the register ID of the name passed in. Used by named register
/// global variables extension. There is no target-independent behaviour
/// so the default action is to bail.
- virtual unsigned getRegisterByName(const char* RegName, EVT VT,
- SelectionDAG &DAG) const {
+ virtual Register getRegisterByName(const char* RegName, LLT Ty,
+ const MachineFunction &MF) const {
report_fatal_error("Named registers not implemented for this target");
}
@@ -3590,13 +4007,25 @@
return Chain;
}
- /// This callback is used to inspect load/store instructions and add
- /// target-specific MachineMemOperand flags to them. The default
- /// implementation does nothing.
- virtual MachineMemOperand::Flags getMMOFlags(const Instruction &I) const {
- return MachineMemOperand::MONone;
+ /// Should SelectionDAG lower an atomic store of the given kind as a normal
+ /// StoreSDNode (as opposed to an AtomicSDNode)? NOTE: The intention is to
+ /// eventually migrate all targets to the using StoreSDNodes, but porting is
+ /// being done target at a time.
+ virtual bool lowerAtomicStoreAsStoreSDNode(const StoreInst &SI) const {
+ assert(SI.isAtomic() && "violated precondition");
+ return false;
}
+ /// Should SelectionDAG lower an atomic load of the given kind as a normal
+ /// LoadSDNode (as opposed to an AtomicSDNode)? NOTE: The intention is to
+ /// eventually migrate all targets to the using LoadSDNodes, but porting is
+ /// being done target at a time.
+ virtual bool lowerAtomicLoadAsLoadSDNode(const LoadInst &LI) const {
+ assert(LI.isAtomic() && "violated precondition");
+ return false;
+ }
+
+
/// This callback is invoked by the type legalizer to legalize nodes with an
/// illegal operand type but legal result types. It replaces the
/// LowerOperation callback in the type Legalizer. The reason we can not do
@@ -3665,6 +4094,7 @@
C_Register, // Constraint represents specific register(s).
C_RegisterClass, // Constraint represents any of register(s) in class.
C_Memory, // Memory constraint.
+ C_Immediate, // Requires an immediate.
C_Other, // Something else.
C_Unknown // Unsupported constraint.
};
@@ -3725,7 +4155,7 @@
/// string itself isn't empty, there was an error parsing.
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL,
const TargetRegisterInfo *TRI,
- ImmutableCallSite CS) const;
+ const CallBase &Call) const;
/// Examine constraint type and operand type and determine a weight value.
/// The operand object must already have been set up with the operand type.
@@ -3762,9 +4192,7 @@
StringRef Constraint, MVT VT) const;
virtual unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const {
- if (ConstraintCode == "i")
- return InlineAsm::Constraint_i;
- else if (ConstraintCode == "m")
+ if (ConstraintCode == "m")
return InlineAsm::Constraint_m;
return InlineAsm::Constraint_Unknown;
}
@@ -3782,7 +4210,7 @@
// Lower custom output constraints. If invalid, return SDValue().
virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag,
- SDLoc DL,
+ const SDLoc &DL,
const AsmOperandInfo &OpInfo,
SelectionDAG &DAG) const;
@@ -3849,6 +4277,22 @@
return SDValue();
}
+ /// Return a target-dependent comparison result if the input operand is
+ /// suitable for use with a square root estimate calculation. For example, the
+ /// comparison may check if the operand is NAN, INF, zero, normal, etc. The
+ /// result should be used as the condition operand for a select or branch.
+ virtual SDValue getSqrtInputTest(SDValue Operand, SelectionDAG &DAG,
+ const DenormalMode &Mode) const {
+ return SDValue();
+ }
+
+ /// Return a target-dependent result if the input operand is not suitable for
+ /// use with a square root estimate calculation.
+ virtual SDValue getSqrtResultForDenormInput(SDValue Operand,
+ SelectionDAG &DAG) const {
+ return DAG.getConstantFP(0.0, SDLoc(Operand), Operand.getValueType());
+ }
+
//===--------------------------------------------------------------------===//
// Legalization utility functions
//
@@ -3863,7 +4307,7 @@
/// \param RL Low bits of the RHS of the MUL. See LL for meaning
/// \param RH High bits of the RHS of the MUL. See LL for meaning.
/// \returns true if the node has been expanded, false if it has not
- bool expandMUL_LOHI(unsigned Opcode, EVT VT, SDLoc dl, SDValue LHS,
+ bool expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, SDValue LHS,
SDValue RHS, SmallVectorImpl<SDValue> &Result, EVT HiLoVT,
SelectionDAG &DAG, MulExpansionKind Kind,
SDValue LL = SDValue(), SDValue LH = SDValue(),
@@ -3891,9 +4335,12 @@
/// Expand rotations.
/// \param N Node to expand
+ /// \param AllowVectorOps expand vector rotate, this should only be performed
+ /// if the legalization is happening outside of LegalizeVectorOps
/// \param Result output after conversion
/// \returns True, if the expansion was successful, false otherwise
- bool expandROT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
+ bool expandROT(SDNode *N, bool AllowVectorOps, SDValue &Result,
+ SelectionDAG &DAG) const;
/// Expand float(f32) to SINT(i64) conversion
/// \param N Node to expand
@@ -3904,18 +4351,27 @@
/// Expand float to UINT conversion
/// \param N Node to expand
/// \param Result output after conversion
+ /// \param Chain output chain after conversion
/// \returns True, if the expansion was successful, false otherwise
- bool expandFP_TO_UINT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
+ bool expandFP_TO_UINT(SDNode *N, SDValue &Result, SDValue &Chain,
+ SelectionDAG &DAG) const;
/// Expand UINT(i64) to double(f64) conversion
/// \param N Node to expand
/// \param Result output after conversion
+ /// \param Chain output chain after conversion
/// \returns True, if the expansion was successful, false otherwise
- bool expandUINT_TO_FP(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
+ bool expandUINT_TO_FP(SDNode *N, SDValue &Result, SDValue &Chain,
+ SelectionDAG &DAG) const;
/// Expand fminnum/fmaxnum into fminnum_ieee/fmaxnum_ieee with quieted inputs.
SDValue expandFMINNUM_FMAXNUM(SDNode *N, SelectionDAG &DAG) const;
+ /// Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
+ /// \param N Node to expand
+ /// \returns The expansion result
+ SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const;
+
/// Expand CTPOP nodes. Expands vector/scalar CTPOP nodes,
/// vector nodes can only succeed if all operations are legal/custom.
/// \param N Node to expand
@@ -3942,17 +4398,20 @@
/// (ABS x) -> (XOR (ADD x, (SRA x, type_size)), (SRA x, type_size))
/// \param N Node to expand
/// \param Result output after conversion
+ /// \param IsNegative indicate negated abs
/// \returns True, if the expansion was successful, false otherwise
- bool expandABS(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
+ bool expandABS(SDNode *N, SDValue &Result, SelectionDAG &DAG,
+ bool IsNegative = false) const;
/// Turn load of vector type into a load of the individual elements.
/// \param LD load to expand
- /// \returns MERGE_VALUEs of the scalar loads with their chains.
- SDValue scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const;
+ /// \returns BUILD_VECTOR and TokenFactor nodes.
+ std::pair<SDValue, SDValue> scalarizeVectorLoad(LoadSDNode *LD,
+ SelectionDAG &DAG) const;
// Turn a store of a vector type into stores of the individual elements.
/// \param ST Store with a vector value type
- /// \returns MERGE_VALUs of the individual store chains.
+ /// \returns TokenFactor of the individual store chains.
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const;
/// Expands an unaligned load to 2 half-size loads for an integer, and
@@ -3982,14 +4441,30 @@
SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT,
SDValue Index) const;
+ /// Method for building the DAG expansion of ISD::[US][MIN|MAX]. This
+ /// method accepts integers as its arguments.
+ SDValue expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const;
+
/// Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT. This
/// method accepts integers as its arguments.
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const;
- /// Method for building the DAG expansion of ISD::SMULFIX. This method accepts
- /// integers as its arguments.
+ /// Method for building the DAG expansion of ISD::[US]SHLSAT. This
+ /// method accepts integers as its arguments.
+ SDValue expandShlSat(SDNode *Node, SelectionDAG &DAG) const;
+
+ /// Method for building the DAG expansion of ISD::[U|S]MULFIX[SAT]. This
+ /// method accepts integers as its arguments.
SDValue expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const;
+ /// Method for building the DAG expansion of ISD::[US]DIVFIX[SAT]. This
+ /// method accepts integers as its arguments.
+ /// Note: This method may fail if the division could not be performed
+ /// within the type. Clients must retry with a wider type if this happens.
+ SDValue expandFixedPointDiv(unsigned Opcode, const SDLoc &dl,
+ SDValue LHS, SDValue RHS,
+ unsigned Scale, SelectionDAG &DAG) const;
+
/// Method for building the DAG expansion of ISD::U(ADD|SUB)O. Expansion
/// always suceeds and populates the Result and Overflow arguments.
void expandUADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow,
@@ -4009,6 +4484,13 @@
/// only the first Count elements of the vector are used.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const;
+ /// Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
+ SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const;
+
+ /// Expand an SREM or UREM using SDIV/UDIV or SDIVREM/UDIVREM, if legal.
+ /// Returns true if the expansion was successful.
+ bool expandREM(SDNode *Node, SDValue &Result, SelectionDAG &DAG) const;
+
//===--------------------------------------------------------------------===//
// Instruction Emitting Hooks
//
@@ -4059,6 +4541,10 @@
// combiner can fold the new nodes.
SDValue lowerCmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG) const;
+ /// Give targets the chance to reduce the number of distinct addresing modes.
+ ISD::MemIndexType getCanonicalIndexType(ISD::MemIndexType IndexType,
+ EVT MemVT, SDValue Offsets) const;
+
private:
SDValue foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
const SDLoc &DL, DAGCombinerInfo &DCI) const;
@@ -4070,6 +4556,11 @@
DAGCombinerInfo &DCI,
const SDLoc &DL) const;
+ // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
+ SDValue optimizeSetCCByHoistingAndByConstFromLogicalShift(
+ EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
+ DAGCombinerInfo &DCI, const SDLoc &DL) const;
+
SDValue prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
SDValue CompTargetNode, ISD::CondCode Cond,
DAGCombinerInfo &DCI, const SDLoc &DL,
@@ -4077,6 +4568,14 @@
SDValue buildUREMEqFold(EVT SETCCVT, SDValue REMNode, SDValue CompTargetNode,
ISD::CondCode Cond, DAGCombinerInfo &DCI,
const SDLoc &DL) const;
+
+ SDValue prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
+ SDValue CompTargetNode, ISD::CondCode Cond,
+ DAGCombinerInfo &DCI, const SDLoc &DL,
+ SmallVectorImpl<SDNode *> &Created) const;
+ SDValue buildSREMEqFold(EVT SETCCVT, SDValue REMNode, SDValue CompTargetNode,
+ ISD::CondCode Cond, DAGCombinerInfo &DCI,
+ const SDLoc &DL) const;
};
/// Given an LLVM IR type and return type attributes, compute the return value