Update prebuilt Clang to r416183b from Android.
https://android.googlesource.com/platform/prebuilts/clang/host/
linux-x86/+/06a71ddac05c22edb2d10b590e1769b3f8619bef
clang 12.0.5 (based on r416183b) from build 7284624.
Change-Id: I277a316abcf47307562d8b748b84870f31a72866
Signed-off-by: Olivier Deprez <olivier.deprez@arm.com>
diff --git a/linux-x64/clang/include/llvm/CodeGen/SelectionDAGNodes.h b/linux-x64/clang/include/llvm/CodeGen/SelectionDAGNodes.h
index 5aab964..3d12240 100644
--- a/linux-x64/clang/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/linux-x64/clang/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -30,6 +30,7 @@
#include "llvm/ADT/iterator_range.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/CodeGen/Register.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DebugLoc.h"
@@ -42,6 +43,7 @@
#include "llvm/Support/Casting.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MachineValueType.h"
+#include "llvm/Support/TypeSize.h"
#include <algorithm>
#include <cassert>
#include <climits>
@@ -83,29 +85,42 @@
/// Node predicates
- /// If N is a BUILD_VECTOR node whose elements are all the same constant or
- /// undefined, return true and return the constant value in \p SplatValue.
- bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);
+/// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the
+/// same constant or undefined, return true and return the constant value in
+/// \p SplatValue.
+bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);
- /// Return true if the specified node is a BUILD_VECTOR where all of the
- /// elements are ~0 or undef.
- bool isBuildVectorAllOnes(const SDNode *N);
+/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
+/// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to
+/// true, it only checks BUILD_VECTOR.
+bool isConstantSplatVectorAllOnes(const SDNode *N,
+ bool BuildVectorOnly = false);
- /// Return true if the specified node is a BUILD_VECTOR where all of the
- /// elements are 0 or undef.
- bool isBuildVectorAllZeros(const SDNode *N);
+/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
+/// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it
+/// only checks BUILD_VECTOR.
+bool isConstantSplatVectorAllZeros(const SDNode *N,
+ bool BuildVectorOnly = false);
- /// Return true if the specified node is a BUILD_VECTOR node of all
- /// ConstantSDNode or undef.
- bool isBuildVectorOfConstantSDNodes(const SDNode *N);
+/// Return true if the specified node is a BUILD_VECTOR where all of the
+/// elements are ~0 or undef.
+bool isBuildVectorAllOnes(const SDNode *N);
- /// Return true if the specified node is a BUILD_VECTOR node of all
- /// ConstantFPSDNode or undef.
- bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);
+/// Return true if the specified node is a BUILD_VECTOR where all of the
+/// elements are 0 or undef.
+bool isBuildVectorAllZeros(const SDNode *N);
- /// Return true if the node has at least one operand and all operands of the
- /// specified node are ISD::UNDEF.
- bool allOperandsUndef(const SDNode *N);
+/// Return true if the specified node is a BUILD_VECTOR node of all
+/// ConstantSDNode or undef.
+bool isBuildVectorOfConstantSDNodes(const SDNode *N);
+
+/// Return true if the specified node is a BUILD_VECTOR node of all
+/// ConstantFPSDNode or undef.
+bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);
+
+/// Return true if the node has at least one operand and all operands of the
+/// specified node are ISD::UNDEF.
+bool allOperandsUndef(const SDNode *N);
} // end namespace ISD
@@ -170,12 +185,16 @@
}
/// Returns the size of the value in bits.
- unsigned getValueSizeInBits() const {
+ ///
+ /// If the value type is a scalable vector type, the scalable property will
+ /// be set and the runtime size will be a positive integer multiple of the
+ /// base size.
+ TypeSize getValueSizeInBits() const {
return getValueType().getSizeInBits();
}
- unsigned getScalarValueSizeInBits() const {
- return getValueType().getScalarType().getSizeInBits();
+ uint64_t getScalarValueSizeInBits() const {
+ return getValueType().getScalarType().getFixedSizeInBits();
}
// Forwarding methods - These forward to the corresponding methods in SDNode.
@@ -351,11 +370,6 @@
/// the backend.
struct SDNodeFlags {
private:
- // This bit is used to determine if the flags are in a defined state.
- // Flag bits can only be masked out during intersection if the masking flags
- // are defined.
- bool AnyDefined : 1;
-
bool NoUnsignedWrap : 1;
bool NoSignedWrap : 1;
bool Exact : 1;
@@ -363,7 +377,6 @@
bool NoInfs : 1;
bool NoSignedZeros : 1;
bool AllowReciprocal : 1;
- bool VectorReduction : 1;
bool AllowContract : 1;
bool ApproximateFuncs : 1;
bool AllowReassociation : 1;
@@ -378,11 +391,10 @@
public:
/// Default constructor turns off all optimization flags.
SDNodeFlags()
- : AnyDefined(false), NoUnsignedWrap(false), NoSignedWrap(false),
- Exact(false), NoNaNs(false), NoInfs(false),
- NoSignedZeros(false), AllowReciprocal(false), VectorReduction(false),
+ : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false),
+ NoInfs(false), NoSignedZeros(false), AllowReciprocal(false),
AllowContract(false), ApproximateFuncs(false),
- AllowReassociation(false), NoFPExcept(true) {}
+ AllowReassociation(false), NoFPExcept(false) {}
/// Propagate the fast-math-flags from an IR FPMathOperator.
void copyFMF(const FPMathOperator &FPMO) {
@@ -395,60 +407,18 @@
setAllowReassociation(FPMO.hasAllowReassoc());
}
- /// Sets the state of the flags to the defined state.
- void setDefined() { AnyDefined = true; }
- /// Returns true if the flags are in a defined state.
- bool isDefined() const { return AnyDefined; }
-
// These are mutators for each flag.
- void setNoUnsignedWrap(bool b) {
- setDefined();
- NoUnsignedWrap = b;
- }
- void setNoSignedWrap(bool b) {
- setDefined();
- NoSignedWrap = b;
- }
- void setExact(bool b) {
- setDefined();
- Exact = b;
- }
- void setNoNaNs(bool b) {
- setDefined();
- NoNaNs = b;
- }
- void setNoInfs(bool b) {
- setDefined();
- NoInfs = b;
- }
- void setNoSignedZeros(bool b) {
- setDefined();
- NoSignedZeros = b;
- }
- void setAllowReciprocal(bool b) {
- setDefined();
- AllowReciprocal = b;
- }
- void setVectorReduction(bool b) {
- setDefined();
- VectorReduction = b;
- }
- void setAllowContract(bool b) {
- setDefined();
- AllowContract = b;
- }
- void setApproximateFuncs(bool b) {
- setDefined();
- ApproximateFuncs = b;
- }
- void setAllowReassociation(bool b) {
- setDefined();
- AllowReassociation = b;
- }
- void setFPExcept(bool b) {
- setDefined();
- NoFPExcept = !b;
- }
+ void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
+ void setNoSignedWrap(bool b) { NoSignedWrap = b; }
+ void setExact(bool b) { Exact = b; }
+ void setNoNaNs(bool b) { NoNaNs = b; }
+ void setNoInfs(bool b) { NoInfs = b; }
+ void setNoSignedZeros(bool b) { NoSignedZeros = b; }
+ void setAllowReciprocal(bool b) { AllowReciprocal = b; }
+ void setAllowContract(bool b) { AllowContract = b; }
+ void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
+ void setAllowReassociation(bool b) { AllowReassociation = b; }
+ void setNoFPExcept(bool b) { NoFPExcept = b; }
// These are accessors for each flag.
bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
@@ -458,22 +428,14 @@
bool hasNoInfs() const { return NoInfs; }
bool hasNoSignedZeros() const { return NoSignedZeros; }
bool hasAllowReciprocal() const { return AllowReciprocal; }
- bool hasVectorReduction() const { return VectorReduction; }
bool hasAllowContract() const { return AllowContract; }
bool hasApproximateFuncs() const { return ApproximateFuncs; }
bool hasAllowReassociation() const { return AllowReassociation; }
- bool hasFPExcept() const { return !NoFPExcept; }
+ bool hasNoFPExcept() const { return NoFPExcept; }
- bool isFast() const {
- return NoSignedZeros && AllowReciprocal && NoNaNs && NoInfs && NoFPExcept &&
- AllowContract && ApproximateFuncs && AllowReassociation;
- }
-
- /// Clear any flags in this flag set that aren't also set in Flags.
- /// If the given Flags are undefined then don't do anything.
+ /// Clear any flags in this flag set that aren't also set in Flags. All
+ /// flags will be cleared if Flags are undefined.
void intersectWith(const SDNodeFlags Flags) {
- if (!Flags.isDefined())
- return;
NoUnsignedWrap &= Flags.NoUnsignedWrap;
NoSignedWrap &= Flags.NoSignedWrap;
Exact &= Flags.Exact;
@@ -481,7 +443,6 @@
NoInfs &= Flags.NoInfs;
NoSignedZeros &= Flags.NoSignedZeros;
AllowReciprocal &= Flags.AllowReciprocal;
- VectorReduction &= Flags.VectorReduction;
AllowContract &= Flags.AllowContract;
ApproximateFuncs &= Flags.ApproximateFuncs;
AllowReassociation &= Flags.AllowReassociation;
@@ -548,16 +509,24 @@
class LSBaseSDNodeBitfields {
friend class LSBaseSDNode;
+ friend class MaskedLoadStoreSDNode;
+ friend class MaskedGatherScatterSDNode;
uint16_t : NumMemSDNodeBits;
- uint16_t AddressingMode : 3; // enum ISD::MemIndexedMode
+ // This storage is shared between disparate class hierarchies to hold an
+ // enumeration specific to the class hierarchy in use.
+ // LSBaseSDNode => enum ISD::MemIndexedMode
+ // MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
+ // MaskedGatherScatterSDNode => enum ISD::MemIndexType
+ uint16_t AddressingMode : 3;
};
enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };
class LoadSDNodeBitfields {
friend class LoadSDNode;
friend class MaskedLoadSDNode;
+ friend class MaskedGatherSDNode;
uint16_t : NumLSBaseSDNodeBits;
@@ -568,6 +537,7 @@
class StoreSDNodeBitfields {
friend class StoreSDNode;
friend class MaskedStoreSDNode;
+ friend class MaskedScatterSDNode;
uint16_t : NumLSBaseSDNodeBits;
@@ -654,6 +624,15 @@
/// \<target\>ISD namespace).
bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
+ /// Test if this node has a target-specific opcode that may raise
+ /// FP exceptions (in the \<target\>ISD namespace and greater than
+ /// FIRST_TARGET_STRICTFP_OPCODE). Note that all target memory
+ /// opcode are currently automatically considered to possibly raise
+ /// FP exceptions as well.
+ bool isTargetStrictFPOpcode() const {
+ return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
+ }
+
/// Test if this node has a target-specific
/// memory-referencing opcode (in the \<target\>ISD namespace and
/// greater than FIRST_TARGET_MEMORY_OPCODE).
@@ -680,32 +659,11 @@
switch (NodeType) {
default:
return false;
- case ISD::STRICT_FADD:
- case ISD::STRICT_FSUB:
- case ISD::STRICT_FMUL:
- case ISD::STRICT_FDIV:
- case ISD::STRICT_FREM:
- case ISD::STRICT_FMA:
- case ISD::STRICT_FSQRT:
- case ISD::STRICT_FPOW:
- case ISD::STRICT_FPOWI:
- case ISD::STRICT_FSIN:
- case ISD::STRICT_FCOS:
- case ISD::STRICT_FEXP:
- case ISD::STRICT_FEXP2:
- case ISD::STRICT_FLOG:
- case ISD::STRICT_FLOG10:
- case ISD::STRICT_FLOG2:
- case ISD::STRICT_FRINT:
- case ISD::STRICT_FNEARBYINT:
- case ISD::STRICT_FMAXNUM:
- case ISD::STRICT_FMINNUM:
- case ISD::STRICT_FCEIL:
- case ISD::STRICT_FFLOOR:
- case ISD::STRICT_FROUND:
- case ISD::STRICT_FTRUNC:
- case ISD::STRICT_FP_ROUND:
- case ISD::STRICT_FP_EXTEND:
+ case ISD::STRICT_FP16_TO_FP:
+ case ISD::STRICT_FP_TO_FP16:
+#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
+ case ISD::STRICT_##DAGN:
+#include "llvm/IR/ConstrainedOps.def"
return true;
}
}
@@ -990,7 +948,6 @@
const SDNodeFlags getFlags() const { return Flags; }
void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }
- bool isFast() { return Flags.isFast(); }
/// Clear any flags in this node that aren't also set in Flags.
/// If Flags is not in a defined state then this has no effect.
@@ -1011,7 +968,11 @@
}
/// Returns MVT::getSizeInBits(getValueType(ResNo)).
- unsigned getValueSizeInBits(unsigned ResNo) const {
+ ///
+ /// If the value type is a scalable vector type, the scalable property will
+ /// be set and the runtime size will be a positive integer multiple of the
+ /// base size.
+ TypeSize getValueSizeInBits(unsigned ResNo) const {
return getValueType(ResNo).getSizeInBits();
}
@@ -1019,6 +980,9 @@
value_iterator value_begin() const { return ValueList; }
value_iterator value_end() const { return ValueList+NumValues; }
+ iterator_range<value_iterator> values() const {
+ return llvm::make_range(value_begin(), value_end());
+ }
/// Return the opcode of this operation for printing.
std::string getOperationName(const SelectionDAG *G = nullptr) const;
@@ -1298,12 +1262,14 @@
bool writeMem() const { return MMO->isStore(); }
/// Returns alignment and volatility of the memory access
- unsigned getOriginalAlignment() const {
- return MMO->getBaseAlignment();
+ Align getOriginalAlign() const { return MMO->getBaseAlign(); }
+ Align getAlign() const { return MMO->getAlign(); }
+ LLVM_ATTRIBUTE_DEPRECATED(unsigned getOriginalAlignment() const,
+ "Use getOriginalAlign() instead") {
+ return MMO->getBaseAlign().value();
}
- unsigned getAlignment() const {
- return MMO->getAlignment();
- }
+ // FIXME: Remove once transition to getAlign is over.
+ unsigned getAlignment() const { return MMO->getAlign().value(); }
/// Return the SubclassData value, without HasDebugValue. This contains an
/// encoding of the volatile flag, as well as bits used by subclasses. This
@@ -1346,6 +1312,17 @@
/// store occurs.
AtomicOrdering getOrdering() const { return MMO->getOrdering(); }
+ /// Return true if the memory operation ordering is Unordered or higher.
+ bool isAtomic() const { return MMO->isAtomic(); }
+
+ /// Returns true if the memory operation doesn't imply any ordering
+ /// constraints on surrounding memory operations beyond the normal memory
+ /// aliasing rules.
+ bool isUnordered() const { return MMO->isUnordered(); }
+
+ /// Returns true if the memory operation is neither atomic or volatile.
+ bool isSimple() const { return !isAtomic() && !isVolatile(); }
+
/// Return the type of the in-memory value.
EVT getMemoryVT() const { return MemoryVT; }
@@ -1371,8 +1348,18 @@
}
const SDValue &getChain() const { return getOperand(0); }
+
const SDValue &getBasePtr() const {
- return getOperand(getOpcode() == ISD::STORE ? 2 : 1);
+ switch (getOpcode()) {
+ case ISD::STORE:
+ case ISD::MSTORE:
+ return getOperand(2);
+ case ISD::MGATHER:
+ case ISD::MSCATTER:
+ return getOperand(3);
+ default:
+ return getOperand(1);
+ }
}
// Methods to support isa and dyn_cast
@@ -1568,6 +1555,8 @@
uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX) {
return Value->getLimitedValue(Limit);
}
+ MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); }
+ Align getAlignValue() const { return Value->getAlignValue(); }
bool isOne() const { return Value->isOne(); }
bool isNullValue() const { return Value->isZero(); }
@@ -1702,16 +1691,16 @@
const GlobalValue *TheGlobal;
int64_t Offset;
- unsigned char TargetFlags;
+ unsigned TargetFlags;
GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
const GlobalValue *GA, EVT VT, int64_t o,
- unsigned char TF);
+ unsigned TF);
public:
const GlobalValue *getGlobal() const { return TheGlobal; }
int64_t getOffset() const { return Offset; }
- unsigned char getTargetFlags() const { return TargetFlags; }
+ unsigned getTargetFlags() const { return TargetFlags; }
// Return the address space this GlobalAddress belongs to.
unsigned getAddressSpace() const;
@@ -1774,20 +1763,46 @@
}
};
+/// This SDNode is used for PSEUDO_PROBE values, which are the function guid and
+/// the index of the basic block being probed. A pseudo probe serves as a place
+/// holder and will be removed at the end of compilation. It does not have any
+/// operand because we do not want the instruction selection to deal with any.
+class PseudoProbeSDNode : public SDNode {
+ friend class SelectionDAG;
+ uint64_t Guid;
+ uint64_t Index;
+ uint32_t Attributes;
+
+ PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl,
+ SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr)
+ : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index),
+ Attributes(Attr) {}
+
+public:
+ uint64_t getGuid() const { return Guid; }
+ uint64_t getIndex() const { return Index; }
+ uint32_t getAttributes() const { return Attributes; }
+
+ // Methods to support isa and dyn_cast
+ static bool classof(const SDNode *N) {
+ return N->getOpcode() == ISD::PSEUDO_PROBE;
+ }
+};
+
class JumpTableSDNode : public SDNode {
friend class SelectionDAG;
int JTI;
- unsigned char TargetFlags;
+ unsigned TargetFlags;
- JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned char TF)
+ JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF)
: SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
}
public:
int getIndex() const { return JTI; }
- unsigned char getTargetFlags() const { return TargetFlags; }
+ unsigned getTargetFlags() const { return TargetFlags; }
static bool classof(const SDNode *N) {
return N->getOpcode() == ISD::JumpTable ||
@@ -1803,23 +1818,23 @@
MachineConstantPoolValue *MachineCPVal;
} Val;
int Offset; // It's a MachineConstantPoolValue if top bit is set.
- unsigned Alignment; // Minimum alignment requirement of CP (not log2 value).
- unsigned char TargetFlags;
+ Align Alignment; // Minimum alignment requirement of CP.
+ unsigned TargetFlags;
ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
- unsigned Align, unsigned char TF)
- : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
- DebugLoc(), getSDVTList(VT)), Offset(o), Alignment(Align),
- TargetFlags(TF) {
+ Align Alignment, unsigned TF)
+ : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
+ DebugLoc(), getSDVTList(VT)),
+ Offset(o), Alignment(Alignment), TargetFlags(TF) {
assert(Offset >= 0 && "Offset is too large");
Val.ConstVal = c;
}
- ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v,
- EVT VT, int o, unsigned Align, unsigned char TF)
- : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
- DebugLoc(), getSDVTList(VT)), Offset(o), Alignment(Align),
- TargetFlags(TF) {
+ ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o,
+ Align Alignment, unsigned TF)
+ : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
+ DebugLoc(), getSDVTList(VT)),
+ Offset(o), Alignment(Alignment), TargetFlags(TF) {
assert(Offset >= 0 && "Offset is too large");
Val.MachineCPVal = v;
Offset |= 1 << (sizeof(unsigned)*CHAR_BIT-1);
@@ -1846,8 +1861,8 @@
// Return the alignment of this constant pool object, which is either 0 (for
// default alignment) or the desired value.
- unsigned getAlignment() const { return Alignment; }
- unsigned char getTargetFlags() const { return TargetFlags; }
+ Align getAlign() const { return Alignment; }
+ unsigned getTargetFlags() const { return TargetFlags; }
Type *getType() const;
@@ -1861,16 +1876,16 @@
class TargetIndexSDNode : public SDNode {
friend class SelectionDAG;
- unsigned char TargetFlags;
+ unsigned TargetFlags;
int Index;
int64_t Offset;
public:
- TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned char TF)
- : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
- TargetFlags(TF), Index(Idx), Offset(Ofs) {}
+ TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF)
+ : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
+ TargetFlags(TF), Index(Idx), Offset(Ofs) {}
- unsigned char getTargetFlags() const { return TargetFlags; }
+ unsigned getTargetFlags() const { return TargetFlags; }
int getIndex() const { return Index; }
int64_t getOffset() const { return Offset; }
@@ -1934,6 +1949,33 @@
/// the vector width and set the bits where elements are undef.
SDValue getSplatValue(BitVector *UndefElements = nullptr) const;
+ /// Find the shortest repeating sequence of values in the build vector.
+ ///
+ /// e.g. { u, X, u, X, u, u, X, u } -> { X }
+ /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
+ ///
+ /// Currently this must be a power-of-2 build vector.
+ /// The DemandedElts mask indicates the elements that must be present,
+ /// undemanded elements in Sequence may be null (SDValue()). If passed a
+ /// non-null UndefElements bitvector, it will resize it to match the original
+ /// vector width and set the bits where elements are undef. If result is
+ /// false, Sequence will be empty.
+ bool getRepeatedSequence(const APInt &DemandedElts,
+ SmallVectorImpl<SDValue> &Sequence,
+ BitVector *UndefElements = nullptr) const;
+
+ /// Find the shortest repeating sequence of values in the build vector.
+ ///
+ /// e.g. { u, X, u, X, u, u, X, u } -> { X }
+ /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
+ ///
+ /// Currently this must be a power-of-2 build vector.
+ /// If passed a non-null UndefElements bitvector, it will resize it to match
+ /// the original vector width and set the bits where elements are undef.
+ /// If result is false, Sequence will be empty.
+ bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
+ BitVector *UndefElements = nullptr) const;
+
/// Returns the demanded splatted constant or null if this is not a constant
/// splat.
///
@@ -2027,13 +2069,13 @@
class RegisterSDNode : public SDNode {
friend class SelectionDAG;
- unsigned Reg;
+ Register Reg;
- RegisterSDNode(unsigned reg, EVT VT)
+ RegisterSDNode(Register reg, EVT VT)
: SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}
public:
- unsigned getReg() const { return Reg; }
+ Register getReg() const { return Reg; }
static bool classof(const SDNode *N) {
return N->getOpcode() == ISD::Register;
@@ -2063,17 +2105,17 @@
const BlockAddress *BA;
int64_t Offset;
- unsigned char TargetFlags;
+ unsigned TargetFlags;
BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
- int64_t o, unsigned char Flags)
+ int64_t o, unsigned Flags)
: SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
BA(ba), Offset(o), TargetFlags(Flags) {}
public:
const BlockAddress *getBlockAddress() const { return BA; }
int64_t getOffset() const { return Offset; }
- unsigned char getTargetFlags() const { return TargetFlags; }
+ unsigned getTargetFlags() const { return TargetFlags; }
static bool classof(const SDNode *N) {
return N->getOpcode() == ISD::BlockAddress ||
@@ -2104,15 +2146,16 @@
friend class SelectionDAG;
const char *Symbol;
- unsigned char TargetFlags;
+ unsigned TargetFlags;
- ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned char TF, EVT VT)
- : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol,
- 0, DebugLoc(), getSDVTList(VT)), Symbol(Sym), TargetFlags(TF) {}
+ ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT)
+ : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0,
+ DebugLoc(), getSDVTList(VT)),
+ Symbol(Sym), TargetFlags(TF) {}
public:
const char *getSymbol() const { return Symbol; }
- unsigned char getTargetFlags() const { return TargetFlags; }
+ unsigned getTargetFlags() const { return TargetFlags; }
static bool classof(const SDNode *N) {
return N->getOpcode() == ISD::ExternalSymbol ||
@@ -2181,8 +2224,6 @@
: MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
LSBaseSDNodeBits.AddressingMode = AM;
assert(getAddressingMode() == AM && "Value truncated");
- assert((!MMO->isAtomic() || MMO->isVolatile()) &&
- "use an AtomicSDNode instead for non-volatile atomics");
}
const SDValue &getOffset() const {
@@ -2272,19 +2313,35 @@
friend class SelectionDAG;
MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
- const DebugLoc &dl, SDVTList VTs, EVT MemVT,
+ const DebugLoc &dl, SDVTList VTs,
+ ISD::MemIndexedMode AM, EVT MemVT,
MachineMemOperand *MMO)
- : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {}
-
- // MaskedLoadSDNode (Chain, ptr, mask, passthru)
- // MaskedStoreSDNode (Chain, data, ptr, mask)
- // Mask is a vector of i1 elements
- const SDValue &getBasePtr() const {
- return getOperand(getOpcode() == ISD::MLOAD ? 1 : 2);
+ : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
+ LSBaseSDNodeBits.AddressingMode = AM;
+ assert(getAddressingMode() == AM && "Value truncated");
}
- const SDValue &getMask() const {
+
+ // MaskedLoadSDNode (Chain, ptr, offset, mask, passthru)
+ // MaskedStoreSDNode (Chain, data, ptr, offset, mask)
+ // Mask is a vector of i1 elements
+ const SDValue &getOffset() const {
return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3);
}
+ const SDValue &getMask() const {
+ return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4);
+ }
+
+ /// Return the addressing mode for this load or store:
+ /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
+ ISD::MemIndexedMode getAddressingMode() const {
+ return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
+ }
+
+ /// Return true if this is a pre/post inc/dec load/store.
+ bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
+
+ /// Return true if this is NOT a pre/post inc/dec load/store.
+ bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
static bool classof(const SDNode *N) {
return N->getOpcode() == ISD::MLOAD ||
@@ -2298,9 +2355,9 @@
friend class SelectionDAG;
MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
- ISD::LoadExtType ETy, bool IsExpanding, EVT MemVT,
- MachineMemOperand *MMO)
- : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, MemVT, MMO) {
+ ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
+ bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
+ : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) {
LoadSDNodeBits.ExtTy = ETy;
LoadSDNodeBits.IsExpanding = IsExpanding;
}
@@ -2310,8 +2367,9 @@
}
const SDValue &getBasePtr() const { return getOperand(1); }
- const SDValue &getMask() const { return getOperand(2); }
- const SDValue &getPassThru() const { return getOperand(3); }
+ const SDValue &getOffset() const { return getOperand(2); }
+ const SDValue &getMask() const { return getOperand(3); }
+ const SDValue &getPassThru() const { return getOperand(4); }
static bool classof(const SDNode *N) {
return N->getOpcode() == ISD::MLOAD;
@@ -2326,9 +2384,9 @@
friend class SelectionDAG;
MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
- bool isTrunc, bool isCompressing, EVT MemVT,
- MachineMemOperand *MMO)
- : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, MemVT, MMO) {
+ ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
+ EVT MemVT, MachineMemOperand *MMO)
+ : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) {
StoreSDNodeBits.IsTruncating = isTrunc;
StoreSDNodeBits.IsCompressing = isCompressing;
}
@@ -2344,9 +2402,10 @@
/// memory at base_addr.
bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }
- const SDValue &getValue() const { return getOperand(1); }
+ const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
- const SDValue &getMask() const { return getOperand(3); }
+ const SDValue &getOffset() const { return getOperand(3); }
+ const SDValue &getMask() const { return getOperand(4); }
static bool classof(const SDNode *N) {
return N->getOpcode() == ISD::MSTORE;
@@ -2362,8 +2421,27 @@
MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
const DebugLoc &dl, SDVTList VTs, EVT MemVT,
- MachineMemOperand *MMO)
- : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {}
+ MachineMemOperand *MMO, ISD::MemIndexType IndexType)
+ : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
+ LSBaseSDNodeBits.AddressingMode = IndexType;
+ assert(getIndexType() == IndexType && "Value truncated");
+ }
+
+ /// How is Index applied to BasePtr when computing addresses.
+ ISD::MemIndexType getIndexType() const {
+ return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
+ }
+ void setIndexType(ISD::MemIndexType IndexType) {
+ LSBaseSDNodeBits.AddressingMode = IndexType;
+ }
+ bool isIndexScaled() const {
+ return (getIndexType() == ISD::SIGNED_SCALED) ||
+ (getIndexType() == ISD::UNSIGNED_SCALED);
+ }
+ bool isIndexSigned() const {
+ return (getIndexType() == ISD::SIGNED_SCALED) ||
+ (getIndexType() == ISD::SIGNED_UNSCALED);
+ }
// In the both nodes address is Op1, mask is Op2:
// MaskedGatherSDNode (Chain, passthru, mask, base, index, scale)
@@ -2387,11 +2465,19 @@
friend class SelectionDAG;
MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
- EVT MemVT, MachineMemOperand *MMO)
- : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO) {}
+ EVT MemVT, MachineMemOperand *MMO,
+ ISD::MemIndexType IndexType, ISD::LoadExtType ETy)
+ : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO,
+ IndexType) {
+ LoadSDNodeBits.ExtTy = ETy;
+ }
const SDValue &getPassThru() const { return getOperand(1); }
+ ISD::LoadExtType getExtensionType() const {
+ return ISD::LoadExtType(LoadSDNodeBits.ExtTy);
+ }
+
static bool classof(const SDNode *N) {
return N->getOpcode() == ISD::MGATHER;
}
@@ -2404,8 +2490,17 @@
friend class SelectionDAG;
MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
- EVT MemVT, MachineMemOperand *MMO)
- : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO) {}
+ EVT MemVT, MachineMemOperand *MMO,
+ ISD::MemIndexType IndexType, bool IsTrunc)
+ : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO,
+ IndexType) {
+ StoreSDNodeBits.IsTruncating = IsTrunc;
+ }
+
+ /// Return true if the op does a truncation before store.
+ /// For integers this is the same as doing a TRUNCATE and storing the result.
+ /// For floats, it is the same as doing an FP_ROUND and storing the result.
+ bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
const SDValue &getValue() const { return getOperand(1); }
@@ -2478,6 +2573,22 @@
}
};
+/// An SDNode that records if a register contains a value that is guaranteed to
+/// be aligned accordingly.
+class AssertAlignSDNode : public SDNode {
+ Align Alignment;
+
+public:
+ AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A)
+ : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {}
+
+ Align getAlign() const { return Alignment; }
+
+ static bool classof(const SDNode *N) {
+ return N->getOpcode() == ISD::AssertAlign;
+ }
+};
+
class SDNodeIterator : public std::iterator<std::forward_iterator_tag,
SDNode, ptrdiff_t> {
const SDNode *Node;
@@ -2539,7 +2650,8 @@
/// with 4 and 8 byte pointer alignment, respectively.
using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode,
BlockAddressSDNode,
- GlobalAddressSDNode>;
+ GlobalAddressSDNode,
+ PseudoProbeSDNode>;
/// The SDNode class with the greatest alignment requirement.
using MostAlignedSDNode = GlobalAddressSDNode;
@@ -2622,6 +2734,16 @@
SDValue LHS, SDValue RHS,
std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
bool AllowUndefs = false, bool AllowTypeMismatch = false);
+
+ /// Returns true if the specified value is the overflow result from one
+ /// of the overflow intrinsic nodes.
+ inline bool isOverflowIntrOpRes(SDValue Op) {
+ unsigned Opc = Op.getOpcode();
+ return (Op.getResNo() == 1 &&
+ (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
+ Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO));
+ }
+
} // end namespace ISD
} // end namespace llvm