From d2220d79bc9067c2e44d5ad602cee37151f8f324 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 30 Dec 2025 12:22:52 -0800
Subject: [PATCH 01/16] Use ValueTypeByHwMode in PatternSortingPredicate.

---
 llvm/utils/TableGen/DAGISelEmitter.cpp | 29 ++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/llvm/utils/TableGen/DAGISelEmitter.cpp b/llvm/utils/TableGen/DAGISelEmitter.cpp
index aaea963d5fde4..111ff6b9ee963 100644
--- a/llvm/utils/TableGen/DAGISelEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelEmitter.cpp
@@ -89,13 +89,30 @@ struct PatternSortingPredicate {
     const TreePatternNode &LT = LHS->getSrcPattern();
     const TreePatternNode &RT = RHS->getSrcPattern();
 
-    MVT LHSVT = LT.getNumTypes() != 0 ? LT.getSimpleType(0) : MVT::Other;
-    MVT RHSVT = RT.getNumTypes() != 0 ? RT.getSimpleType(0) : MVT::Other;
-    if (LHSVT.isVector() != RHSVT.isVector())
-      return RHSVT.isVector();
+    bool LHSIsVector = false;
+    bool RHSIsVector = false;
+    bool LHSIsFP = false;
+    bool RHSIsFP = false;
+
+    if (LT.getNumTypes() != 0) {
+      for (auto VT : LT.getType(0)) {
+        LHSIsVector |= VT.second.isVector();
+        LHSIsFP |= VT.second.isFloatingPoint();
+      }
+    }
+
+    if (RT.getNumTypes() != 0) {
+      for (auto VT : RT.getType(0)) {
+        RHSIsVector |= VT.second.isVector();
+        RHSIsFP |= VT.second.isFloatingPoint();
+      }
+    }
+
+    if (LHSIsVector != RHSIsVector)
+      return RHSIsVector;
 
-    if (LHSVT.isFloatingPoint() != RHSVT.isFloatingPoint())
-      return RHSVT.isFloatingPoint();
+    if (LHSIsFP != RHSIsFP)
+      return RHSIsFP;
 
     // Otherwise, if the patterns might both match, sort based on complexity,
     // which means that we prefer to match patterns that cover more nodes in the

From 0649d2cd2efb817c627a57eb0cda841932c7f9d0 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 30 Dec 2025 15:06:25 -0800
Subject: [PATCH 02/16] Use VVT in getPatternSize.

---
 llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
index 35f8a06916298..42f06ab8fd3a4 100644
--- a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
@@ -1480,10 +1480,9 @@ static unsigned getPatternSize(const TreePatternNode &P,
   // Count children in the count if they are also nodes.
   for (const TreePatternNode &Child : P.children()) {
     if (!Child.isLeaf() && Child.getNumTypes()) {
-      const TypeSetByHwMode &T0 = Child.getExtType(0);
-      // At this point, all variable type sets should be simple, i.e. only
-      // have a default mode.
-      if (T0.getMachineValueType() != MVT::Other) {
+      // FIXME: Can we assume non-simple VTs should be counted?
+      auto VVT = Child.getType(0);
+      if (llvm::any_of(VVT, [](auto &P) { return P.second != MVT::Other; })) {
         Size += getPatternSize(Child, CGP);
         continue;
       }

From 2ca3b8149404256d628ece2921fd86c75bc71582 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 30 Dec 2025 15:06:47 -0800
Subject: [PATCH 03/16] Move SignExtend64 for EmitInteger to isel.

---
 llvm/include/llvm/CodeGen/SelectionDAGISel.h  |  20 ++
 .../CodeGen/SelectionDAG/SelectionDAGISel.cpp |   1 +
 .../TableGen/Common/CodeGenDAGPatterns.cpp    |   5 +-
 .../TableGen/Common/CodeGenDAGPatterns.h      |   2 +-
 llvm/utils/TableGen/Common/DAGISelMatcher.cpp |  33 +--
 llvm/utils/TableGen/Common/DAGISelMatcher.h   |  58 ++---
 llvm/utils/TableGen/DAGISelEmitter.cpp        |   2 +-
 llvm/utils/TableGen/DAGISelMatcherEmitter.cpp | 227 ++++++++++++------
 llvm/utils/TableGen/DAGISelMatcherGen.cpp     |  21 +-
 llvm/utils/TableGen/DAGISelMatcherOpt.cpp     |   9 +-
 10 files changed, 248 insertions(+), 130 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
index 569353670c532..cad00a6b62ab5 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
@@ -202,7 +202,9 @@ class SelectionDAGISel {
     // Space-optimized forms that implicitly encode VT.
     OPC_CheckTypeI32,
     OPC_CheckTypeI64,
+    OPC_CheckTypeByHwMode,
     OPC_CheckTypeRes,
+    OPC_CheckTypeResByHwMode,
     OPC_SwitchType,
     OPC_CheckChild0Type,
     OPC_CheckChild1Type,
@@ -231,6 +233,15 @@ class SelectionDAGISel {
     OPC_CheckChild6TypeI64,
     OPC_CheckChild7TypeI64,
 
+    OPC_CheckChild0TypeByHwMode,
+    OPC_CheckChild1TypeByHwMode,
+    OPC_CheckChild2TypeByHwMode,
+    OPC_CheckChild3TypeByHwMode,
+    OPC_CheckChild4TypeByHwMode,
+    OPC_CheckChild5TypeByHwMode,
+    OPC_CheckChild6TypeByHwMode,
+    OPC_CheckChild7TypeByHwMode,
+
     OPC_CheckInteger,
     OPC_CheckChild0Integer,
     OPC_CheckChild1Integer,
@@ -261,10 +272,13 @@ class SelectionDAGISel {
     OPC_EmitIntegerI16,
     OPC_EmitIntegerI32,
     OPC_EmitIntegerI64,
+    OPC_EmitIntegerByHwMode,
     OPC_EmitRegister,
     OPC_EmitRegisterI32,
     OPC_EmitRegisterI64,
+    OPC_EmitRegisterByHwMode,
     OPC_EmitRegister2,
+    OPC_EmitRegisterByHwMode2,
     OPC_EmitConvertToTarget,
     OPC_EmitConvertToTarget0,
     OPC_EmitConvertToTarget1,
@@ -290,6 +304,7 @@ class SelectionDAGISel {
     OPC_EmitCopyToRegTwoByte,
     OPC_EmitNodeXForm,
     OPC_EmitNode,
+    OPC_EmitNodeByHwMode,
     // Space-optimized forms that implicitly encode number of result VTs.
     OPC_EmitNode0,
     OPC_EmitNode1,
@@ -302,6 +317,7 @@ class SelectionDAGISel {
     OPC_EmitNode1Chain,
     OPC_EmitNode2Chain,
     OPC_MorphNodeTo,
+    OPC_MorphNodeToByHwMode,
     // Space-optimized forms that implicitly encode number of result VTs.
     OPC_MorphNodeTo0,
     OPC_MorphNodeTo1,
@@ -448,6 +464,10 @@ class SelectionDAGISel {
     llvm_unreachable("Tblgen should generate this!");
   }
 
+  virtual MVT getValueTypeByHwMode(unsigned Index) const {
+    llvm_unreachable("Tblgen should generate the implementation of this!");
+  }
+
   void SelectCodeCommon(SDNode *NodeToMatch, const uint8_t *MatcherTable,
                         unsigned TableSize);
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 91c4a37d9885c..5b4b5e12904ec 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -3920,6 +3920,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
         break;
       }
       int64_t Val = GetSignedVBR(MatcherTable, MatcherIndex);
+      Val = SignExtend64(Val, MVT(VT).getFixedSizeInBits());
       RecordedNodes.emplace_back(
           CurDAG->getSignedConstant(Val, SDLoc(NodeToMatch), VT,
                                     /*isTarget=*/true),
diff --git a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
index 42f06ab8fd3a4..7dbd75cf65382 100644
--- a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
@@ -3318,7 +3318,7 @@ void TreePattern::dump() const { print(errs()); }
 // CodeGenDAGPatterns implementation
 //
 
-CodeGenDAGPatterns::CodeGenDAGPatterns(const RecordKeeper &R)
+CodeGenDAGPatterns::CodeGenDAGPatterns(const RecordKeeper &R, bool ExpandHwMode)
     : Records(R), Target(R), Intrinsics(R),
       LegalVTS(Target.getLegalValueTypes()),
       LegalPtrVTS(ComputeLegalPtrTypes()) {
@@ -3338,7 +3338,8 @@ CodeGenDAGPatterns::CodeGenDAGPatterns(const RecordKeeper &R)
   // Break patterns with parameterized types into a series of patterns,
   // where each one has a fixed type and is predicated on the conditions
   // of the associated HW mode.
-  ExpandHwModeBasedTypes();
+  if (ExpandHwMode)
+    ExpandHwModeBasedTypes();
 
   // Infer instruction flags.  For example, we can detect loads,
   // stores, and side effects in many cases by examining an
diff --git a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h
index 220fa43bf5037..667cffc4efbd6 100644
--- a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h
+++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h
@@ -1123,7 +1123,7 @@ class CodeGenDAGPatterns {
   unsigned NumScopes = 0;
 
 public:
-  CodeGenDAGPatterns(const RecordKeeper &R);
+  CodeGenDAGPatterns(const RecordKeeper &R, bool ExpandHwMode = true);
 
   CodeGenTarget &getTargetInfo() { return Target; }
   const CodeGenTarget &getTargetInfo() const { return Target; }
diff --git a/llvm/utils/TableGen/Common/DAGISelMatcher.cpp b/llvm/utils/TableGen/Common/DAGISelMatcher.cpp
index 768e80f39a028..f46388d67af33 100644
--- a/llvm/utils/TableGen/Common/DAGISelMatcher.cpp
+++ b/llvm/utils/TableGen/Common/DAGISelMatcher.cpp
@@ -180,8 +180,7 @@ void SwitchOpcodeMatcher::printImpl(raw_ostream &OS, indent Indent) const {
 }
 
 void CheckTypeMatcher::printImpl(raw_ostream &OS, indent Indent) const {
-  OS << Indent << "CheckType " << getEnumName(Type) << ", ResNo=" << ResNo
-     << '\n';
+  OS << Indent << "CheckType " << Type << ", ResNo=" << ResNo << '\n';
 }
 
 void SwitchTypeMatcher::printImpl(raw_ostream &OS, indent Indent) const {
@@ -194,8 +193,7 @@ void SwitchTypeMatcher::printImpl(raw_ostream &OS, indent Indent) const {
 }
 
 void CheckChildTypeMatcher::printImpl(raw_ostream &OS, indent Indent) const {
-  OS << Indent << "CheckChildType " << ChildNo << " " << getEnumName(Type)
-     << '\n';
+  OS << Indent << "CheckChildType " << ChildNo << " " << Type << '\n';
 }
 
 void CheckIntegerMatcher::printImpl(raw_ostream &OS, indent Indent) const {
@@ -245,7 +243,7 @@ void CheckImmAllZerosVMatcher::printImpl(raw_ostream &OS, indent Indent) const {
 }
 
 void EmitIntegerMatcher::printImpl(raw_ostream &OS, indent Indent) const {
-  OS << Indent << "EmitInteger " << Val << " VT=" << getEnumName(VT) << '\n';
+  OS << Indent << "EmitInteger " << Val << " VT=" << VT << '\n';
 }
 
 void EmitRegisterMatcher::printImpl(raw_ostream &OS, indent Indent) const {
@@ -254,7 +252,7 @@ void EmitRegisterMatcher::printImpl(raw_ostream &OS, indent Indent) const {
     OS << Reg->getName();
   else
     OS << "zero_reg";
-  OS << " VT=" << getEnumName(VT) << '\n';
+  OS << " VT=" << VT << '\n';
 }
 
 void EmitConvertToTargetMatcher::printImpl(raw_ostream &OS,
@@ -281,8 +279,8 @@ void EmitNodeMatcherCommon::printImpl(raw_ostream &OS, indent Indent) const {
   OS << (isa<MorphNodeToMatcher>(this) ? "MorphNodeTo: " : "EmitNode: ")
      << CGI.Namespace << "::" << CGI.getName() << ": <todo flags> ";
 
-  for (MVT VT : VTs)
-    OS << ' ' << getEnumName(VT);
+  for (const ValueTypeByHwMode &VT : VTs)
+    OS << ' ' << VT;
   OS << '(';
   for (unsigned Operand : Operands)
     OS << Operand << ' ';
@@ -316,19 +314,26 @@ void MorphNodeToMatcher::anchor() {}
 
 // isContradictoryImpl Implementations.
 
-static bool TypesAreContradictory(MVT T1, MVT T2) {
+static bool TypesAreContradictory(const ValueTypeByHwMode &VT1,
+                                  const ValueTypeByHwMode &VT2) {
   // If the two types are the same, then they are the same, so they don't
   // contradict.
-  if (T1 == T2)
+  if (VT1 == VT2)
     return false;
 
+  if (!VT1.isSimple() || !VT2.isSimple())
+    return false;
+
+  MVT T1 = VT1.getSimple();
+  MVT T2 = VT2.getSimple();
+
   if (T1 == MVT::pAny)
-    return TypesAreContradictory(MVT::iPTR, T2) &&
-           TypesAreContradictory(MVT::cPTR, T2);
+    return TypesAreContradictory(MVT(MVT::iPTR), T2) &&
+           TypesAreContradictory(MVT(MVT::cPTR), T2);
 
   if (T2 == MVT::pAny)
-    return TypesAreContradictory(T1, MVT::iPTR) &&
-           TypesAreContradictory(T1, MVT::cPTR);
+    return TypesAreContradictory(T1, MVT(MVT::iPTR)) &&
+           TypesAreContradictory(T1, MVT(MVT::cPTR));
 
   // If either type is about iPtr, then they don't conflict unless the other
   // one is not a scalar integer type.
diff --git a/llvm/utils/TableGen/Common/DAGISelMatcher.h b/llvm/utils/TableGen/Common/DAGISelMatcher.h
index 192d5c47d3489..6c073f48a9cbe 100644
--- a/llvm/utils/TableGen/Common/DAGISelMatcher.h
+++ b/llvm/utils/TableGen/Common/DAGISelMatcher.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_UTILS_TABLEGEN_COMMON_DAGISELMATCHER_H
 #define LLVM_UTILS_TABLEGEN_COMMON_DAGISELMATCHER_H
 
+#include "InfoByHwMode.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -516,14 +517,14 @@ class SwitchOpcodeMatcher : public Matcher {
 /// CheckTypeMatcher - This checks to see if the current node has the
 /// specified type at the specified result, if not it fails to match.
 class CheckTypeMatcher : public Matcher {
-  MVT Type;
+  ValueTypeByHwMode Type;
   unsigned ResNo;
 
 public:
-  CheckTypeMatcher(MVT type, unsigned resno)
-      : Matcher(CheckType), Type(type), ResNo(resno) {}
+  CheckTypeMatcher(ValueTypeByHwMode type, unsigned resno)
+      : Matcher(CheckType), Type(std::move(type)), ResNo(resno) {}
 
-  MVT getType() const { return Type; }
+  const ValueTypeByHwMode &getType() const { return Type; }
   unsigned getResNo() const { return ResNo; }
 
   static bool classof(const Matcher *N) { return N->getKind() == CheckType; }
@@ -565,14 +566,14 @@ class SwitchTypeMatcher : public Matcher {
 /// specified type, if not it fails to match.
 class CheckChildTypeMatcher : public Matcher {
   unsigned ChildNo;
-  MVT Type;
+  ValueTypeByHwMode Type;
 
 public:
-  CheckChildTypeMatcher(unsigned childno, MVT type)
-      : Matcher(CheckChildType), ChildNo(childno), Type(type) {}
+  CheckChildTypeMatcher(unsigned childno, ValueTypeByHwMode type)
+      : Matcher(CheckChildType), ChildNo(childno), Type(std::move(type)) {}
 
   unsigned getChildNo() const { return ChildNo; }
-  MVT getType() const { return Type; }
+  const ValueTypeByHwMode &getType() const { return Type; }
 
   static bool classof(const Matcher *N) {
     return N->getKind() == CheckChildType;
@@ -831,22 +832,20 @@ class EmitIntegerMatcher : public Matcher {
   // Optional string to give the value a symbolic name for readability.
   std::string Str;
   int64_t Val;
-  MVT VT;
+  ValueTypeByHwMode VT;
 
   unsigned ResultNo;
 
 public:
-  EmitIntegerMatcher(int64_t val, MVT vt, unsigned resultNo)
-      : Matcher(EmitInteger),
-        Val(SignExtend64(val, MVT(vt).getFixedSizeInBits())), VT(vt),
-        ResultNo(resultNo) {}
+  EmitIntegerMatcher(int64_t val, ValueTypeByHwMode vt, unsigned resultNo)
+      : Matcher(EmitInteger), Val(val), VT(std::move(vt)), ResultNo(resultNo) {}
   EmitIntegerMatcher(const std::string &str, int64_t val, MVT vt,
                      unsigned resultNo)
       : Matcher(EmitInteger), Str(str), Val(val), VT(vt), ResultNo(resultNo) {}
 
   const std::string &getString() const { return Str; }
   int64_t getValue() const { return Val; }
-  MVT getVT() const { return VT; }
+  const ValueTypeByHwMode &getVT() const { return VT; }
   unsigned getResultNo() const { return ResultNo; }
 
   static bool classof(const Matcher *N) { return N->getKind() == EmitInteger; }
@@ -865,16 +864,18 @@ class EmitRegisterMatcher : public Matcher {
   /// Reg - The def for the register that we're emitting.  If this is null, then
   /// this is a reference to zero_reg.
   const CodeGenRegister *Reg;
-  MVT VT;
+  ValueTypeByHwMode VT;
 
   unsigned ResultNo;
 
 public:
-  EmitRegisterMatcher(const CodeGenRegister *reg, MVT vt, unsigned resultNo)
-      : Matcher(EmitRegister), Reg(reg), VT(vt), ResultNo(resultNo) {}
+  EmitRegisterMatcher(const CodeGenRegister *reg, ValueTypeByHwMode vt,
+                      unsigned resultNo)
+      : Matcher(EmitRegister), Reg(reg), VT(std::move(vt)), ResultNo(resultNo) {
+  }
 
   const CodeGenRegister *getReg() const { return Reg; }
-  MVT getVT() const { return VT; }
+  const ValueTypeByHwMode &getVT() const { return VT; }
   unsigned getResultNo() const { return ResultNo; }
 
   static bool classof(const Matcher *N) { return N->getKind() == EmitRegister; }
@@ -1002,7 +1003,7 @@ class EmitNodeXFormMatcher : public Matcher {
 /// MorphNodeTo.
 class EmitNodeMatcherCommon : public Matcher {
   const CodeGenInstruction &CGI;
-  const SmallVector<MVT, 3> VTs;
+  const SmallVector<ValueTypeByHwMode, 3> VTs;
   const SmallVector<unsigned, 6> Operands;
   bool HasChain, HasInGlue, HasOutGlue, HasMemRefs;
 
@@ -1012,7 +1013,8 @@ class EmitNodeMatcherCommon : public Matcher {
   int NumFixedArityOperands;
 
 public:
-  EmitNodeMatcherCommon(const CodeGenInstruction &cgi, ArrayRef<MVT> vts,
+  EmitNodeMatcherCommon(const CodeGenInstruction &cgi,
+                        ArrayRef<ValueTypeByHwMode> vts,
                         ArrayRef<unsigned> operands, bool hasChain,
                         bool hasInGlue, bool hasOutGlue, bool hasmemrefs,
                         int numfixedarityoperands, bool isMorphNodeTo)
@@ -1024,7 +1026,7 @@ class EmitNodeMatcherCommon : public Matcher {
   const CodeGenInstruction &getInstruction() const { return CGI; }
 
   unsigned getNumVTs() const { return VTs.size(); }
-  MVT getVT(unsigned i) const {
+  const ValueTypeByHwMode &getVT(unsigned i) const {
     assert(i < VTs.size());
     return VTs[i];
   }
@@ -1035,8 +1037,8 @@ class EmitNodeMatcherCommon : public Matcher {
     return Operands[i];
   }
 
-  const SmallVectorImpl<MVT> &getVTList() const { return VTs; }
-  const SmallVectorImpl<unsigned> &getOperandList() const { return Operands; }
+  ArrayRef<ValueTypeByHwMode> getVTList() const { return VTs; }
+  ArrayRef<unsigned> getOperandList() const { return Operands; }
 
   bool hasChain() const { return HasChain; }
   bool hasInGlue() const { return HasInGlue; }
@@ -1059,9 +1061,10 @@ class EmitNodeMatcher : public EmitNodeMatcherCommon {
   unsigned FirstResultSlot;
 
 public:
-  EmitNodeMatcher(const CodeGenInstruction &cgi, ArrayRef<MVT> vts,
-                  ArrayRef<unsigned> operands, bool hasChain, bool hasInGlue,
-                  bool hasOutGlue, bool hasmemrefs, int numfixedarityoperands,
+  EmitNodeMatcher(const CodeGenInstruction &cgi,
+                  ArrayRef<ValueTypeByHwMode> vts, ArrayRef<unsigned> operands,
+                  bool hasChain, bool hasInGlue, bool hasOutGlue,
+                  bool hasmemrefs, int numfixedarityoperands,
                   unsigned firstresultslot)
       : EmitNodeMatcherCommon(cgi, vts, operands, hasChain, hasInGlue,
                               hasOutGlue, hasmemrefs, numfixedarityoperands,
@@ -1078,7 +1081,8 @@ class MorphNodeToMatcher : public EmitNodeMatcherCommon {
   const PatternToMatch &Pattern;
 
 public:
-  MorphNodeToMatcher(const CodeGenInstruction &cgi, ArrayRef<MVT> vts,
+  MorphNodeToMatcher(const CodeGenInstruction &cgi,
+                     ArrayRef<ValueTypeByHwMode> vts,
                      ArrayRef<unsigned> operands, bool hasChain, bool hasInGlue,
                      bool hasOutGlue, bool hasmemrefs,
                      int numfixedarityoperands, const PatternToMatch &pattern)
diff --git a/llvm/utils/TableGen/DAGISelEmitter.cpp b/llvm/utils/TableGen/DAGISelEmitter.cpp
index 111ff6b9ee963..7540298080b58 100644
--- a/llvm/utils/TableGen/DAGISelEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelEmitter.cpp
@@ -30,7 +30,7 @@ class DAGISelEmitter {
   const CodeGenDAGPatterns CGP;
 
 public:
-  explicit DAGISelEmitter(const RecordKeeper &R) : Records(R), CGP(R) {}
+  explicit DAGISelEmitter(const RecordKeeper &R) : Records(R), CGP(R, false) {}
   void run(raw_ostream &OS);
 };
 } // End anonymous namespace
diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index 7877a1f36424f..8325ee2b25d53 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -71,6 +71,8 @@ class MatcherTableEmitter {
   std::vector<std::string> VecIncludeStrings;
   MapVector<std::string, unsigned, StringMap<unsigned>> VecPatterns;
 
+  std::map<ValueTypeByHwMode, unsigned> ValueTypeMap;
+
   unsigned getPatternIdxFromTable(std::string &&P, std::string &&include_loc) {
     const auto [It, Inserted] =
         VecPatterns.try_emplace(std::move(P), VecPatterns.size());
@@ -174,6 +176,8 @@ class MatcherTableEmitter {
 
   void EmitPredicateFunctions(raw_ostream &OS);
 
+  void EmitValueTypeFunction(raw_ostream &OS);
+
   void EmitHistogram(const Matcher *N, raw_ostream &OS);
 
   void EmitPatternMatchTable(raw_ostream &OS);
@@ -212,6 +216,13 @@ class MatcherTableEmitter {
     }
     return Entry - 1;
   }
+
+  unsigned getValueTypeID(const ValueTypeByHwMode &VT) {
+    unsigned &Entry = ValueTypeMap[VT];
+    if (Entry == 0)
+      Entry = ValueTypeMap.size();
+    return Entry - 1;
+  }
 };
 } // end anonymous namespace.
 
@@ -660,46 +671,68 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
   }
 
   case Matcher::CheckType: {
+    const ValueTypeByHwMode &VTBH = cast<CheckTypeMatcher>(N)->getType();
     if (cast<CheckTypeMatcher>(N)->getResNo() == 0) {
-      MVT VT = cast<CheckTypeMatcher>(N)->getType();
+      if (VTBH.isSimple()) {
+        MVT VT = VTBH.getSimple();
+        switch (VT.SimpleTy) {
+        case MVT::i32:
+        case MVT::i64:
+          OS << "OPC_CheckTypeI" << MVT(VT).getSizeInBits() << ",\n";
+          return 1;
+        default:
+          OS << "OPC_CheckType, ";
+          if (!OmitComments)
+            OS << "/*" << getEnumName(VT) << "*/";
+          unsigned NumBytes = EmitVBRValue(VT.SimpleTy, OS);
+          OS << '\n';
+          return NumBytes + 1;
+        }
+      } else {
+        OS << "OPC_CheckTypeByHwMode, ";
+        unsigned NumBytes = EmitVBRValue(getValueTypeID(VTBH), OS);
+        OS << '\n';
+        return NumBytes + 1;
+      }
+    }
+    if (VTBH.isSimple()) {
+      OS << "OPC_CheckTypeRes, " << cast<CheckTypeMatcher>(N)->getResNo()
+         << ", ";
+      if (!OmitComments)
+        OS << "/*" << getEnumName(VTBH.getSimple()) << "*/";
+      unsigned NumBytes = EmitVBRValue(VTBH.getSimple().SimpleTy, OS);
+      OS << '\n';
+      return NumBytes + 2;
+    }
+    unsigned NumBytes = EmitVBRValue(getValueTypeID(VTBH), OS);
+    OS << '\n';
+    return NumBytes + 2;
+  }
+
+  case Matcher::CheckChildType: {
+    const ValueTypeByHwMode &VTBH = cast<CheckChildTypeMatcher>(N)->getType();
+    if (VTBH.isSimple()) {
+      MVT VT = VTBH.getSimple();
       switch (VT.SimpleTy) {
       case MVT::i32:
       case MVT::i64:
-        OS << "OPC_CheckTypeI" << MVT(VT).getSizeInBits() << ",\n";
+        OS << "OPC_CheckChild" << cast<CheckChildTypeMatcher>(N)->getChildNo()
+           << "TypeI" << MVT(VT).getSizeInBits() << ",\n";
         return 1;
       default:
-        OS << "OPC_CheckType, ";
+        OS << "OPC_CheckChild" << cast<CheckChildTypeMatcher>(N)->getChildNo()
+           << "Type, ";
         if (!OmitComments)
           OS << "/*" << getEnumName(VT) << "*/";
         unsigned NumBytes = EmitVBRValue(VT.SimpleTy, OS);
-        OS << "\n";
+        OS << '\n';
         return NumBytes + 1;
       }
-    }
-    OS << "OPC_CheckTypeRes, " << cast<CheckTypeMatcher>(N)->getResNo() << ", ";
-    if (!OmitComments)
-      OS << "/*" << getEnumName(cast<CheckTypeMatcher>(N)->getType()) << "*/";
-    unsigned NumBytes =
-        EmitVBRValue(cast<CheckTypeMatcher>(N)->getType().SimpleTy, OS);
-    OS << "\n";
-    return NumBytes + 2;
-  }
-
-  case Matcher::CheckChildType: {
-    MVT VT = cast<CheckChildTypeMatcher>(N)->getType();
-    switch (VT.SimpleTy) {
-    case MVT::i32:
-    case MVT::i64:
-      OS << "OPC_CheckChild" << cast<CheckChildTypeMatcher>(N)->getChildNo()
-         << "TypeI" << MVT(VT).getSizeInBits() << ",\n";
-      return 1;
-    default:
+    } else {
       OS << "OPC_CheckChild" << cast<CheckChildTypeMatcher>(N)->getChildNo()
-         << "Type, ";
-      if (!OmitComments)
-        OS << "/*" << getEnumName(VT) << "*/";
-      unsigned NumBytes = EmitVBRValue(VT.SimpleTy, OS);
-      OS << "\n";
+         << "TypeByHwMode, ";
+      unsigned NumBytes = EmitVBRValue(getValueTypeID(VTBH), OS);
+      OS << '\n';
       return NumBytes + 1;
     }
   }
@@ -796,22 +829,29 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
     const auto *IM = cast<EmitIntegerMatcher>(N);
     int64_t Val = IM->getValue();
     const std::string &Str = IM->getString();
-    MVT VT = IM->getVT();
+    const ValueTypeByHwMode &VTBH = IM->getVT();
     unsigned TypeBytes = 0;
-    switch (VT.SimpleTy) {
-    case MVT::i8:
-    case MVT::i16:
-    case MVT::i32:
-    case MVT::i64:
-      OS << "OPC_EmitIntegerI" << VT.getSizeInBits() << ", ";
-      break;
-    default:
-      OS << "OPC_EmitInteger, ";
-      if (!OmitComments)
-        OS << "/*" << getEnumName(VT) << "*/";
-      TypeBytes = EmitVBRValue(VT.SimpleTy, OS);
+    if (VTBH.isSimple()) {
+      MVT VT = VTBH.getSimple();
+      switch (VT.SimpleTy) {
+      case MVT::i8:
+      case MVT::i16:
+      case MVT::i32:
+      case MVT::i64:
+        OS << "OPC_EmitIntegerI" << VT.getSizeInBits() << ", ";
+        break;
+      default:
+        OS << "OPC_EmitInteger, ";
+        if (!OmitComments)
+          OS << "/*" << getEnumName(VT) << "*/";
+        TypeBytes = EmitVBRValue(VT.SimpleTy, OS);
+        OS << ' ';
+        break;
+      }
+    } else {
+      OS << "OPC_EmitIntegerByHwMode, ";
+      TypeBytes = EmitVBRValue(getValueTypeID(VTBH), OS);
       OS << ' ';
-      break;
     }
     // If the value is 63 or smaller, use the string directly. Otherwise, use
     // a VBR.
@@ -834,30 +874,45 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
   case Matcher::EmitRegister: {
     const EmitRegisterMatcher *Matcher = cast<EmitRegisterMatcher>(N);
     const CodeGenRegister *Reg = Matcher->getReg();
-    MVT VT = Matcher->getVT();
+    const ValueTypeByHwMode &VTBH = Matcher->getVT();
     unsigned OpBytes;
-    // If the enum value of the register is larger than one byte can handle,
-    // use EmitRegister2.
-    if (Reg && Reg->EnumValue > 255) {
-      OS << "OPC_EmitRegister2, ";
-      if (!OmitComments)
-        OS << "/*" << getEnumName(VT) << "*/";
-      OpBytes = EmitVBRValue(VT.SimpleTy, OS);
-      OS << "TARGET_VAL(" << getQualifiedName(Reg->TheDef) << "),\n";
-      return OpBytes + 3;
-    }
-    switch (VT.SimpleTy) {
-    case MVT::i32:
-    case MVT::i64:
-      OpBytes = 1;
-      OS << "OPC_EmitRegisterI" << VT.getSizeInBits() << ", ";
-      break;
-    default:
-      OS << "OPC_EmitRegister, ";
-      if (!OmitComments)
-        OS << "/*" << getEnumName(VT) << "*/";
-      OpBytes = EmitVBRValue(VT.SimpleTy, OS) + 1;
-      break;
+    if (VTBH.isSimple()) {
+      MVT VT = VTBH.getSimple();
+      // If the enum value of the register is larger than one byte can handle,
+      // use EmitRegister2.
+      if (Reg && Reg->EnumValue > 255) {
+        OS << "OPC_EmitRegister2, ";
+        if (!OmitComments)
+          OS << "/*" << getEnumName(VT) << "*/";
+        OpBytes = EmitVBRValue(VT.SimpleTy, OS);
+        OS << " TARGET_VAL(" << getQualifiedName(Reg->TheDef) << "),\n";
+        return OpBytes + 3;
+      }
+      switch (VT.SimpleTy) {
+      case MVT::i32:
+      case MVT::i64:
+        OpBytes = 1;
+        OS << "OPC_EmitRegisterI" << VT.getSizeInBits() << ", ";
+        break;
+      default:
+        OS << "OPC_EmitRegister, ";
+        if (!OmitComments)
+          OS << "/*" << getEnumName(VT) << "*/";
+        OpBytes = EmitVBRValue(VT.SimpleTy, OS) + 1;
+        OS << ' ';
+        break;
+      }
+    } else {
+      if (Reg && Reg->EnumValue > 255) {
+        OS << "OPC_EmitRegisterByHwMode2, ";
+        OpBytes = EmitVBRValue(getValueTypeID(VTBH), OS);
+        OS << " TARGET_VAL(" << getQualifiedName(Reg->TheDef) << "),\n";
+        return OpBytes + 3;
+      }
+
+      OS << "OPC_EmitRegisterByHwMode, ";
+      OpBytes = EmitVBRValue(getValueTypeID(VTBH), OS) + 1;
+      OS << ' ';
     }
     if (Reg) {
       OS << getQualifiedName(Reg->TheDef);
@@ -964,9 +1019,15 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
       OS << "OPC_CaptureDeactivationSymbol,\n";
       OS.indent(FullIndexWidth + Indent);
     }
+
+    bool ByHwMode =
+        llvm::any_of(EN->getVTList(), [](const ValueTypeByHwMode &VT) {
+          return !VT.isSimple();
+        });
+
     bool IsEmitNode = isa<EmitNodeMatcher>(EN);
     OS << (IsEmitNode ? "OPC_EmitNode" : "OPC_MorphNodeTo");
-    bool CompressVTs = EN->getNumVTs() < 3;
+    bool CompressVTs = !ByHwMode && EN->getNumVTs() < 3;
     bool CompressNodeInfo = false;
     if (CompressVTs) {
       OS << EN->getNumVTs();
@@ -991,6 +1052,9 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
       }
     }
 
+    if (ByHwMode)
+      OS << "ByHwMode";
+
     const CodeGenInstruction &CGI = EN->getInstruction();
     OS << ", TARGET_VAL(" << CGI.Namespace << "::" << CGI.TheDef->getName()
        << ")";
@@ -1018,11 +1082,18 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
       OS << ",";
     }
     unsigned NumTypeBytes = 0;
-    for (unsigned i = 0, e = EN->getNumVTs(); i != e; ++i) {
-      OS << ' ';
-      if (!OmitComments)
-        OS << "/*" << getEnumName(EN->getVT(i)) << "*/";
-      NumTypeBytes += EmitVBRValue(EN->getVT(i).SimpleTy, OS);
+    if (ByHwMode) {
+      for (unsigned i = 0, e = EN->getNumVTs(); i != e; ++i) {
+        OS << " 0,";
+        ++NumTypeBytes;
+      }
+    } else {
+      for (unsigned i = 0, e = EN->getNumVTs(); i != e; ++i) {
+        OS << ' ';
+        if (!OmitComments)
+          OS << "/*" << getEnumName(EN->getVT(i).getSimple()) << "*/";
+        NumTypeBytes += EmitVBRValue(EN->getVT(i).getSimple().SimpleTy, OS);
+      }
     }
 
     OS << ' ' << EN->getNumOperands();
@@ -1267,6 +1338,18 @@ void MatcherTableEmitter::EmitPredicateFunctions(raw_ostream &OS) {
   }
 }
 
+void MatcherTableEmitter::EmitValueTypeFunction(raw_ostream &OS) {
+  if (ValueTypeMap.empty())
+    return;
+
+  BeginEmitFunction(OS, "MVT", "getValueTypeByHwMode(unsigned Index) const",
+                    /*AddOverride=*/true);
+  OS << "{\n";
+
+  OS << "}\n";
+  EndEmitFunction(OS);
+}
+
 static StringRef getOpcodeString(Matcher::KindTy Kind) {
   switch (Kind) {
   case Matcher::Scope:
@@ -1429,6 +1512,8 @@ void llvm::EmitMatcherTable(Matcher *TheMatcher, const CodeGenDAGPatterns &CGP,
   // Next up, emit the function for node and pattern predicates:
   MatcherEmitter.EmitPredicateFunctions(OS);
 
+  MatcherEmitter.EmitValueTypeFunction(OS);
+
   if (InstrumentCoverage)
     MatcherEmitter.EmitPatternMatchTable(OS);
 
diff --git a/llvm/utils/TableGen/DAGISelMatcherGen.cpp b/llvm/utils/TableGen/DAGISelMatcherGen.cpp
index dd4aefad02af7..14cf59ba2ea68 100644
--- a/llvm/utils/TableGen/DAGISelMatcherGen.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherGen.cpp
@@ -536,7 +536,7 @@ void MatcherGen::EmitMatchCode(const TreePatternNode &N,
   }
 
   for (unsigned I : ResultsToTypeCheck)
-    AddMatcher(new CheckTypeMatcher(N.getSimpleType(I), I));
+    AddMatcher(new CheckTypeMatcher(N.getType(I), I));
 }
 
 /// EmitMatcherCode - Generate the code that matches the predicate of this
@@ -660,7 +660,7 @@ void MatcherGen::EmitResultLeafAsOperand(const TreePatternNode &N,
   assert(N.isLeaf() && "Must be a leaf");
 
   if (const IntInit *II = dyn_cast<IntInit>(N.getLeafValue())) {
-    AddMatcher(new EmitIntegerMatcher(II->getValue(), N.getSimpleType(0),
+    AddMatcher(new EmitIntegerMatcher(II->getValue(), N.getType(0),
                                       NextRecordedOperandNo));
     ResultOps.push_back(NextRecordedOperandNo++);
     return;
@@ -671,21 +671,21 @@ void MatcherGen::EmitResultLeafAsOperand(const TreePatternNode &N,
     const Record *Def = DI->getDef();
     if (Def->isSubClassOf("Register")) {
       const CodeGenRegister *Reg = CGP.getTargetInfo().getRegBank().getReg(Def);
-      AddMatcher(new EmitRegisterMatcher(Reg, N.getSimpleType(0),
-                                         NextRecordedOperandNo));
+      AddMatcher(
+          new EmitRegisterMatcher(Reg, N.getType(0), NextRecordedOperandNo));
       ResultOps.push_back(NextRecordedOperandNo++);
       return;
     }
 
     if (Def->getName() == "zero_reg") {
-      AddMatcher(new EmitRegisterMatcher(nullptr, N.getSimpleType(0),
+      AddMatcher(new EmitRegisterMatcher(nullptr, N.getType(0),
                                          NextRecordedOperandNo));
       ResultOps.push_back(NextRecordedOperandNo++);
       return;
     }
 
     if (Def->getName() == "undef_tied_input") {
-      MVT ResultVT = N.getSimpleType(0);
+      ValueTypeByHwMode ResultVT = N.getType(0);
       auto IDOperandNo = NextRecordedOperandNo++;
       const Record *ImpDef = Def->getRecords().getDef("IMPLICIT_DEF");
       const CodeGenInstruction &II = CGP.getTargetInfo().getInstruction(ImpDef);
@@ -879,9 +879,9 @@ void MatcherGen::EmitResultInstructionAsOperand(
   // Result order: node results, chain, glue
 
   // Determine the result types.
-  SmallVector<MVT, 4> ResultVTs;
+  SmallVector<ValueTypeByHwMode, 4> ResultVTs;
   for (unsigned i = 0, e = N.getNumTypes(); i != e; ++i)
-    ResultVTs.push_back(N.getSimpleType(i));
+    ResultVTs.push_back(N.getType(i));
 
   // If this is the root instruction of a pattern that has physical registers in
   // its result pattern, add output VTs for them.  For example, X86 has:
@@ -956,8 +956,9 @@ void MatcherGen::EmitResultInstructionAsOperand(
                                  NumFixedArityOperands, NextRecordedOperandNo));
 
   // The non-chain and non-glue results of the newly emitted node get recorded.
-  for (MVT ResultVT : ResultVTs) {
-    if (ResultVT == MVT::Other || ResultVT == MVT::Glue)
+  for (const ValueTypeByHwMode &ResultVT : ResultVTs) {
+    if (ResultVT.isSimple() && (ResultVT.getSimple() == MVT::Other ||
+                                ResultVT.getSimple() == MVT::Glue))
       break;
     OutputOps.push_back(NextRecordedOperandNo++);
   }
diff --git a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp
index af73eb216041e..bcd7e4821bdca 100644
--- a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp
@@ -282,7 +282,7 @@ static void ContractNodes(std::unique_ptr<Matcher> &InputMatcherPtr,
 #endif
 
         if (ResultsMatch) {
-          ArrayRef<MVT> VTs = EN->getVTList();
+          ArrayRef<ValueTypeByHwMode> VTs = EN->getVTList();
           ArrayRef<unsigned> Operands = EN->getOperandList();
           MatcherPtr->reset(new MorphNodeToMatcher(
               EN->getInstruction(), VTs, Operands, EN->hasChain(),
@@ -518,10 +518,11 @@ static void FactorScope(std::unique_ptr<Matcher> &MatcherPtr) {
     if (AllTypeChecks) {
       CheckTypeMatcher *CTM = cast_or_null<CheckTypeMatcher>(
           FindNodeWithKind(Optn, Matcher::CheckType));
-      if (!CTM ||
+      if (!CTM || !CTM->getType().isSimple() ||
           // iPTR/cPTR checks could alias any other case without us knowing,
           // don't bother with them.
-          CTM->getType() == MVT::iPTR || CTM->getType() == MVT::cPTR ||
+          CTM->getType().getSimple() == MVT::iPTR ||
+          CTM->getType().getSimple() == MVT::cPTR ||
           // SwitchType only works for result #0.
           CTM->getResNo() != 0 ||
           // If the CheckType isn't at the start of the list, see if we can move
@@ -563,7 +564,7 @@ static void FactorScope(std::unique_ptr<Matcher> &MatcherPtr) {
 
       auto *CTM = cast<CheckTypeMatcher>(M);
       Matcher *MatcherWithoutCTM = Optn->unlinkNode(CTM);
-      MVT CTMTy = CTM->getType();
+      MVT CTMTy = CTM->getType().getSimple();
       delete CTM;
 
       unsigned &Entry = TypeEntry[CTMTy.SimpleTy];

From be086a300c86283768e39cd3515b92558a106be5 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 5 Jan 2026 11:47:01 -0800
Subject: [PATCH 04/16] [SelectionDAG][WIP] Move HwMode expansion from tablegen
 to SelectionISel.

The way HwMode is currently implemented, tablegen duplicates each
pattern that is dependent on hardware mode. The HwMode predicate is
added as a pattern predicate on the duplicated pattern.

RISC-V uses HwMode on the GPR register class which means almost every
isel pattern is affected by HwMode. This results in the isel table
being nearly twice the size it would be if we only had a single GPR size.

This patch proposes to do the expansion at instruction selection time
instead. To accomplish this new opcodes like OPC_CheckTypeByHwMode
are added to the isel table. The unique combinations of types and HwMode
are converted to an index that is the payload for the new opcodes.
TableGen emits a new virtual function getValueTypeByHwMode that uses
this index and the current HwMode to look up the type.

This reduces the size of the isel table on RISC-V from ~2.38 million bytes
to ~1.38 million bytes.

I did not add an OPC_SwitchTypeByHwMode opcode yet. If the VT requires a
hardware mode, we emit an OPC_Scope+OPC_CheckTypeByHwMode instead. I
expect adding an OPC_SwitchTypeByHwMode could further reduce the table
size. I will investigate this as a follow up.

I haven't measured yet, but it's possible the new getValueTypeByHwMode
may have an affect on compile time. If necessary we could add a cache on
this lookup to mitigate some impact.

Many of the matcher classes in tablegen now use ValueTypeByHwMode instead
of MVT. This may have an impact on the memory usage and runtime of tablegen.
We can mitigate some of this by splitting the matchers into MVT and
ValueTypeByHwMode versions. We can also explore alternate data structures
for ValueTypeByHwMode instead of a std::map. Maybe a sorted vector.

I plan to do some cleanup to try to reduce some code duplication, but
I wanted to get early feedback on the direction.

Given the scope of this patch and the timing of the LLVM 22 branch,
I don't expect to commit this until after the branch.
---
 llvm/include/llvm/CodeGen/SelectionDAGISel.h  |   2 +
 .../CodeGen/SelectionDAG/SelectionDAGISel.cpp | 118 +++++++++++++++++-
 llvm/test/TableGen/RegClassByHwMode.td        |  49 +-------
 .../TableGen/Common/CodeGenDAGPatterns.cpp    |   4 +-
 .../TableGen/Common/CodeGenDAGPatterns.h      |   5 +-
 llvm/utils/TableGen/DAGISelMatcherEmitter.cpp |  47 ++++++-
 6 files changed, 167 insertions(+), 58 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
index cad00a6b62ab5..c96a7ba97201f 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
@@ -77,6 +77,8 @@ class SelectionDAGISel {
   bool MatchFilterFuncName = false;
   StringRef FuncName;
 
+  unsigned HwMode;
+
   explicit SelectionDAGISel(TargetMachine &tm,
                             CodeGenOptLevel OL = CodeGenOptLevel::Default);
   virtual ~SelectionDAGISel();
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 5b4b5e12904ec..0ff95264af417 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -531,6 +531,8 @@ void SelectionDAGISel::initializeAnalysisResults(
   SP = &FAM.getResult<SSPLayoutAnalysis>(Fn);
 
   TTI = &FAM.getResult<TargetIRAnalysis>(Fn);
+
+  HwMode = MF->getSubtarget().getHwMode();
 }
 
 void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) {
@@ -588,6 +590,8 @@ void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) {
   SP = &MFP.getAnalysis<StackProtector>().getLayoutInfo();
 
   TTI = &MFP.getAnalysis<TargetTransformInfoWrapperPass>().getTTI(Fn);
+
+  HwMode = MF->getSubtarget().getHwMode();
 }
 
 bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
@@ -2747,6 +2751,18 @@ getSimpleVT(const uint8_t *MatcherTable, unsigned &MatcherIndex) {
   return static_cast<MVT::SimpleValueType>(SimpleVT);
 }
 
+/// getSimpleVT - Decode a value in MatcherTable, if it's a VBR encoded value,
+/// use GetVBR to decode it.
+LLVM_ATTRIBUTE_ALWAYS_INLINE static MVT
+getHwModeVT(const uint8_t *MatcherTable, unsigned &MatcherIndex,
+                   const SelectionDAGISel &SDISel) {
+  unsigned Index = MatcherTable[MatcherIndex++];
+  if (Index & 128)
+     Index = GetVBR(Index, MatcherTable, MatcherIndex);
+
+  return SDISel.getValueTypeByHwMode(Index);
+}
+
 void SelectionDAGISel::Select_JUMP_TABLE_DEBUG_INFO(SDNode *N) {
   SDLoc dl(N);
   CurDAG->SelectNodeTo(N, TargetOpcode::JUMP_TABLE_DEBUG_INFO, MVT::Glue,
@@ -3150,12 +3166,23 @@ static unsigned IsPredicateKnownToFail(
     Result = !::CheckType(VT, N, SDISel.TLI, SDISel.CurDAG->getDataLayout());
     return Index;
   }
+  case SelectionDAGISel::OPC_CheckTypeByHwMode: {
+    MVT VT = getHwModeVT(Table, Index, SDISel);
+    Result = !::CheckType(VT.SimpleTy, N, SDISel.TLI, SDISel.CurDAG->getDataLayout());
+    return Index;
+  }
   case SelectionDAGISel::OPC_CheckTypeRes: {
     unsigned Res = Table[Index++];
     Result = !::CheckType(getSimpleVT(Table, Index), N.getValue(Res),
                           SDISel.TLI, SDISel.CurDAG->getDataLayout());
     return Index;
   }
+  case SelectionDAGISel::OPC_CheckTypeResByHwMode: {
+    unsigned Res = Table[Index++];
+    MVT VT = getHwModeVT(Table, Index, SDISel);
+    Result = !::CheckType(VT.SimpleTy, N.getValue(Res), SDISel.TLI, SDISel.CurDAG->getDataLayout());
+    return Index;
+  }
   case SelectionDAGISel::OPC_CheckChild0Type:
   case SelectionDAGISel::OPC_CheckChild1Type:
   case SelectionDAGISel::OPC_CheckChild2Type:
@@ -3198,6 +3225,20 @@ static unsigned IsPredicateKnownToFail(
                                SDISel.CurDAG->getDataLayout(), ChildNo);
     return Index;
   }
+  case SelectionDAGISel::OPC_CheckChild0TypeByHwMode:
+  case SelectionDAGISel::OPC_CheckChild1TypeByHwMode:
+  case SelectionDAGISel::OPC_CheckChild2TypeByHwMode:
+  case SelectionDAGISel::OPC_CheckChild3TypeByHwMode:
+  case SelectionDAGISel::OPC_CheckChild4TypeByHwMode:
+  case SelectionDAGISel::OPC_CheckChild5TypeByHwMode:
+  case SelectionDAGISel::OPC_CheckChild6TypeByHwMode:
+  case SelectionDAGISel::OPC_CheckChild7TypeByHwMode: {
+    MVT VT = getHwModeVT(Table, Index, SDISel);
+    unsigned ChildNo = Opcode - SelectionDAGISel::OPC_CheckChild0TypeByHwMode;
+    Result = !::CheckChildType(VT.SimpleTy, N, SDISel.TLI,
+                               SDISel.CurDAG->getDataLayout(), ChildNo);
+    return Index;
+  }
   case SelectionDAGISel::OPC_CheckCondCode:
     Result = !::CheckCondCode(Table, Index, N);
     return Index;
@@ -3718,6 +3759,12 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
         break;
       continue;
     }
+    case OPC_CheckTypeByHwMode: {
+      MVT VT = getHwModeVT(MatcherTable, MatcherIndex, *this);
+      if (!::CheckType(VT.SimpleTy, N, TLI, CurDAG->getDataLayout()))
+        break;
+      continue;
+    }
 
     case OPC_CheckTypeRes: {
       unsigned Res = MatcherTable[MatcherIndex++];
@@ -3726,6 +3773,13 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
         break;
       continue;
     }
+    case OPC_CheckTypeResByHwMode: {
+      unsigned Res = MatcherTable[MatcherIndex++];
+      MVT VT = getHwModeVT(MatcherTable, MatcherIndex, *this);
+      if (!::CheckType(VT.SimpleTy, N.getValue(Res), TLI, CurDAG->getDataLayout()))
+        break;
+      continue;
+    }
 
     case OPC_SwitchOpcode: {
       unsigned CurNodeOpcode = N.getOpcode();
@@ -3832,6 +3886,20 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
         break;
       continue;
     }
+    case OPC_CheckChild0TypeByHwMode:
+    case OPC_CheckChild1TypeByHwMode:
+    case OPC_CheckChild2TypeByHwMode:
+    case OPC_CheckChild3TypeByHwMode:
+    case OPC_CheckChild4TypeByHwMode:
+    case OPC_CheckChild5TypeByHwMode:
+    case OPC_CheckChild6TypeByHwMode:
+    case OPC_CheckChild7TypeByHwMode: {
+      MVT VT = getHwModeVT(MatcherTable, MatcherIndex, *this);
+      unsigned ChildNo = Opcode - OPC_CheckChild0TypeByHwMode;
+      if (!::CheckChildType(VT.SimpleTy, N, TLI, CurDAG->getDataLayout(), ChildNo))
+        break;
+      continue;
+    }
     case OPC_CheckCondCode:
       if (!::CheckCondCode(MatcherTable, MatcherIndex, N)) break;
       continue;
@@ -3927,6 +3995,17 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
           nullptr);
       continue;
     }
+    case OPC_EmitIntegerByHwMode: {
+      MVT VT = getHwModeVT(MatcherTable, MatcherIndex, *this);
+      int64_t Val = GetSignedVBR(MatcherTable, MatcherIndex);
+      Val = SignExtend64(Val, MVT(VT).getFixedSizeInBits());
+      RecordedNodes.emplace_back(
+          CurDAG->getSignedConstant(Val, SDLoc(NodeToMatch), VT.SimpleTy,
+                                    /*isTarget=*/true),
+          nullptr);
+      continue;
+    }
+
     case OPC_EmitRegister:
     case OPC_EmitRegisterI32:
     case OPC_EmitRegisterI64: {
@@ -3946,6 +4025,12 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
       RecordedNodes.emplace_back(CurDAG->getRegister(RegNo, VT), nullptr);
       continue;
     }
+    case OPC_EmitRegisterByHwMode: {
+      MVT VT = getHwModeVT(MatcherTable, MatcherIndex, *this);
+      unsigned RegNo = MatcherTable[MatcherIndex++];
+      RecordedNodes.emplace_back(CurDAG->getRegister(RegNo, VT), nullptr);
+      continue;
+    }
     case OPC_EmitRegister2: {
       // For targets w/ more than 256 register names, the register enum
       // values are stored in two bytes in the matcher table (just like
@@ -3956,6 +4041,16 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
       RecordedNodes.emplace_back(CurDAG->getRegister(RegNo, VT), nullptr);
       continue;
     }
+    case OPC_EmitRegisterByHwMode2: {
+      // For targets w/ more than 256 register names, the register enum
+      // values are stored in two bytes in the matcher table (just like
+      // opcodes).
+      MVT VT = getHwModeVT(MatcherTable, MatcherIndex, *this);
+      unsigned RegNo = MatcherTable[MatcherIndex++];
+      RegNo |= MatcherTable[MatcherIndex++] << 8;
+      RecordedNodes.emplace_back(CurDAG->getRegister(RegNo, VT), nullptr);
+      continue;
+    }
 
     case OPC_EmitConvertToTarget:
     case OPC_EmitConvertToTarget0:
@@ -4115,6 +4210,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
     }
 
     case OPC_EmitNode:
+    case OPC_EmitNodeByHwMode:
     case OPC_EmitNode0:
     case OPC_EmitNode1:
     case OPC_EmitNode2:
@@ -4125,6 +4221,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
     case OPC_EmitNode1Chain:
     case OPC_EmitNode2Chain:
     case OPC_MorphNodeTo:
+    case OPC_MorphNodeToByHwMode:
     case OPC_MorphNodeTo0:
     case OPC_MorphNodeTo1:
     case OPC_MorphNodeTo2:
@@ -4188,11 +4285,20 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
       else
         NumVTs = MatcherTable[MatcherIndex++];
       SmallVector<EVT, 4> VTs;
-      for (unsigned i = 0; i != NumVTs; ++i) {
-        MVT::SimpleValueType VT = getSimpleVT(MatcherTable, MatcherIndex);
-        if (VT == MVT::iPTR)
-          VT = TLI->getPointerTy(CurDAG->getDataLayout()).SimpleTy;
-        VTs.push_back(VT);
+      if (Opcode == OPC_EmitNodeByHwMode || Opcode == OPC_MorphNodeToByHwMode) {
+        for (unsigned i = 0; i != NumVTs; ++i) {
+          MVT VT = getHwModeVT(MatcherTable, MatcherIndex, *this);
+          if (VT == MVT::iPTR)
+            VT = TLI->getPointerTy(CurDAG->getDataLayout());
+          VTs.push_back(VT);
+        }
+      } else {
+        for (unsigned i = 0; i != NumVTs; ++i) {
+          MVT::SimpleValueType VT = getSimpleVT(MatcherTable, MatcherIndex);
+          if (VT == MVT::iPTR)
+            VT = TLI->getPointerTy(CurDAG->getDataLayout()).SimpleTy;
+          VTs.push_back(VT);
+        }
       }
 
       if (EmitNodeInfo & OPFL_Chain)
@@ -4259,7 +4365,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
       // Create the node.
       MachineSDNode *Res = nullptr;
       bool IsMorphNodeTo =
-          Opcode == OPC_MorphNodeTo ||
+          Opcode == OPC_MorphNodeTo || Opcode == OPC_MorphNodeToByHwMode ||
           (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2GlueOutput);
       if (!IsMorphNodeTo) {
         // If this is a normal EmitNode command, just create the new node and
diff --git a/llvm/test/TableGen/RegClassByHwMode.td b/llvm/test/TableGen/RegClassByHwMode.td
index 193a4c616bb89..fa23932a10ef9 100644
--- a/llvm/test/TableGen/RegClassByHwMode.td
+++ b/llvm/test/TableGen/RegClassByHwMode.td
@@ -193,30 +193,8 @@ include "Common/RegClassByHwModeCommon.td"
 // ISEL-SDAG-NEXT: OPC_RecordNode, // #0 = 'st' chained node
 // ISEL-SDAG-NEXT: OPC_RecordChild1, // #1 = $val
 // ISEL-SDAG-NEXT: OPC_RecordChild2, // #2 = $src
-// ISEL-SDAG-NEXT: OPC_Scope, {{[0-9]+}}, /*->{{[0-9]+}}*/ // 2 children in Scope
-// ISEL-SDAG-NEXT: OPC_CheckChild2TypeI32,
 // ISEL-SDAG-NEXT: OPC_CheckPredicate0,  // Predicate_unindexedstore
 // ISEL-SDAG-NEXT: OPC_CheckPredicate1,  // Predicate_store
-// ISEL-SDAG-NEXT: OPC_Scope, {{[0-9]+}}, /*->{{[0-9]+}}*/ // 3 children in Scope
-// ISEL-SDAG-NEXT: OPC_CheckPatternPredicate0, // (Subtarget->hasAlignedRegisters())
-// ISEL-SDAG-NEXT: OPC_EmitMergeInputChains1_0,
-// ISEL-SDAG-NEXT: OPC_MorphNodeTo0, TARGET_VAL(MyTarget::MY_STORE), 0|OPFL_Chain|OPFL_MemRefs,
-
-// ISEL-SDAG: /*Scope*/
-// ISEL-SDAG: OPC_CheckPatternPredicate1, // (Subtarget->hasUnalignedRegisters())
-// ISEL-SDAG-NEXT: OPC_EmitMergeInputChains1_0,
-// ISEL-SDAG-NEXT: OPC_MorphNodeTo0, TARGET_VAL(MyTarget::MY_STORE), 0|OPFL_Chain|OPFL_MemRefs,
-
-// ISEL-SDAG: /*Scope*/
-// ISEL-SDAG: OPC_CheckPatternPredicate2, // !((Subtarget->hasAlignedRegisters())) && !((Subtarget->hasUnalignedRegisters())) && !((Subtarget->isPtr64()))
-// ISEL-SDAG-NEXT: OPC_EmitMergeInputChains1_0,
-// ISEL-SDAG-NEXT: OPC_MorphNodeTo0, TARGET_VAL(MyTarget::MY_STORE), 0|OPFL_Chain|OPFL_MemRefs,
-
-// ISEL-SDAG: /*Scope*/
-// ISEL-SDAG-NEXT: OPC_CheckChild2TypeI64,
-// ISEL-SDAG-NEXT: OPC_CheckPredicate0,  // Predicate_unindexedstore
-// ISEL-SDAG-NEXT: OPC_CheckPredicate1,  // Predicate_store
-// ISEL-SDAG-NEXT: OPC_CheckPatternPredicate3, // (Subtarget->isPtr64())
 // ISEL-SDAG-NEXT: OPC_EmitMergeInputChains1_0,
 // ISEL-SDAG-NEXT: OPC_MorphNodeTo0, TARGET_VAL(MyTarget::MY_STORE), 0|OPFL_Chain|OPFL_MemRefs,
 
@@ -224,33 +202,12 @@ include "Common/RegClassByHwModeCommon.td"
 // ISEL-SDAG-NEXT: OPC_RecordMemRef,
 // ISEL-SDAG-NEXT: OPC_RecordNode, // #0 = 'ld' chained node
 // ISEL-SDAG-NEXT: OPC_RecordChild1, // #1 = $src
-// ISEL-SDAG-NEXT: OPC_CheckTypeI64,
-// ISEL-SDAG-NEXT: OPC_Scope, {{[0-9]+}}, /*->{{[0-9]+}}*/ // 2 children in Scope
-// ISEL-SDAG-NEXT: OPC_CheckChild1TypeI32,
-// ISEL-SDAG-NEXT: OPC_CheckPredicate2,  // Predicate_unindexedload
-// ISEL-SDAG-NEXT: OPC_CheckPredicate3,  // Predicate_load
-// ISEL-SDAG-NEXT: OPC_Scope, {{[0-9]+}}, /*->{{[0-9]+}}*/ // 3 children in Scope
-// ISEL-SDAG-NEXT: OPC_CheckPatternPredicate0, // (Subtarget->hasAlignedRegisters())
-// ISEL-SDAG-NEXT: OPC_EmitMergeInputChains1_0,
-// ISEL-SDAG-NEXT: OPC_MorphNodeTo1, TARGET_VAL(MyTarget::MY_LOAD), 0|OPFL_Chain|OPFL_MemRefs,
-
-// ISEL-SDAG: /*Scope*/
-// ISEL-SDAG: OPC_CheckPatternPredicate1, // (Subtarget->hasUnalignedRegisters())
-// ISEL-SDAG-NEXT: OPC_EmitMergeInputChains1_0,
-// ISEL-SDAG-NEXT: OPC_MorphNodeTo1, TARGET_VAL(MyTarget::MY_LOAD), 0|OPFL_Chain|OPFL_MemRefs,
-
-// ISEL-SDAG: /*Scope*/
-// ISEL-SDAG: OPC_CheckPatternPredicate2, // !((Subtarget->hasAlignedRegisters())) && !((Subtarget->hasUnalignedRegisters())) && !((Subtarget->isPtr64()))
-// ISEL-SDAG-NEXT: OPC_EmitMergeInputChains1_0,
-// ISEL-SDAG-NEXT: OPC_MorphNodeTo1, TARGET_VAL(MyTarget::MY_LOAD), 0|OPFL_Chain|OPFL_MemRefs,
-
-// ISEL-SDAG: /*Scope*/
-// ISEL-SDAG-NEXT: OPC_CheckChild1TypeI64,
+// ISEL-SDAG-NEXT: OPC_CheckChild1TypeByHwMode, /*{(*:i32),(m3:i64)}*/0,
 // ISEL-SDAG-NEXT: OPC_CheckPredicate2,  // Predicate_unindexedload
 // ISEL-SDAG-NEXT: OPC_CheckPredicate3,  // Predicate_load
-// ISEL-SDAG-NEXT: OPC_CheckPatternPredicate3, // (Subtarget->isPtr64())
+// ISEL-SDAG-NEXT: OPC_CheckTypeI64,
 // ISEL-SDAG-NEXT: OPC_EmitMergeInputChains1_0,
-// ISEL-SDAG-NEXT: OPC_MorphNodeTo1, TARGET_VAL(MyTarget::MY_LOAD), 0|OPFL_Chain|OPFL_MemRefs,
+// ISEL-SDAG-NEXT: OPC_MorphNodeToByHwMode, TARGET_VAL(MyTarget::MY_LOAD), 0|OPFL_Chain|OPFL_MemRefs,
 
 
 
diff --git a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
index 7dbd75cf65382..af1b5592e56c9 100644
--- a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
@@ -94,13 +94,15 @@ bool TypeSetByHwMode::isValueTypeByHwMode(bool AllowEmpty) const {
   return true;
 }
 
-ValueTypeByHwMode TypeSetByHwMode::getValueTypeByHwMode() const {
+ValueTypeByHwMode TypeSetByHwMode::getValueTypeByHwMode(bool SkipEmpty) const {
   assert(isValueTypeByHwMode(true) &&
          "The type set has multiple types for at least one HW mode");
   ValueTypeByHwMode VVT;
   VVT.PtrAddrSpace = AddrSpace;
 
   for (const auto &I : *this) {
+    if (SkipEmpty && I.second.empty())
+      continue;
     MVT T = I.second.empty() ? MVT::Other : *I.second.begin();
     VVT.insertTypeForMode(I.first, T);
   }
diff --git a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h
index 667cffc4efbd6..7d93e9ce126d5 100644
--- a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h
+++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h
@@ -190,7 +190,7 @@ struct TypeSetByHwMode : public InfoByHwMode<MachineValueTypeSet> {
   SetType &getOrCreate(unsigned Mode) { return Map[Mode]; }
 
   bool isValueTypeByHwMode(bool AllowEmpty) const;
-  ValueTypeByHwMode getValueTypeByHwMode() const;
+  ValueTypeByHwMode getValueTypeByHwMode(bool SkipEmpty = false) const;
 
   LLVM_ATTRIBUTE_ALWAYS_INLINE
   bool isMachineValueType() const {
@@ -672,6 +672,9 @@ class TreePatternNode : public RefCountedBase<TreePatternNode> {
 
   // Type accessors.
   unsigned getNumTypes() const { return Types.size(); }
+  ValueTypeByHwMode getType(unsigned ResNo) const {
+    return Types[ResNo].getValueTypeByHwMode(/*SkipEmpty=*/true);
+  }
   const std::vector<TypeSetByHwMode> &getExtTypes() const { return Types; }
   const TypeSetByHwMode &getExtType(unsigned ResNo) const {
     return Types[ResNo];
diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index 8325ee2b25d53..17259831634b5 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -690,6 +690,8 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
         }
       } else {
         OS << "OPC_CheckTypeByHwMode, ";
+        if (!OmitComments)
+          OS << "/*" << VTBH << "*/";
         unsigned NumBytes = EmitVBRValue(getValueTypeID(VTBH), OS);
         OS << '\n';
         return NumBytes + 1;
@@ -704,6 +706,10 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
       OS << '\n';
       return NumBytes + 2;
     }
+    OS << "OPC_CheckTypeResByHwMode, " << cast<CheckTypeMatcher>(N)->getResNo()
+       << ", ";
+    if (!OmitComments)
+      OS << "/*" << VTBH << "*/";
     unsigned NumBytes = EmitVBRValue(getValueTypeID(VTBH), OS);
     OS << '\n';
     return NumBytes + 2;
@@ -731,6 +737,8 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
     } else {
       OS << "OPC_CheckChild" << cast<CheckChildTypeMatcher>(N)->getChildNo()
          << "TypeByHwMode, ";
+      if (!OmitComments)
+        OS << "/*" << VTBH << "*/";
       unsigned NumBytes = EmitVBRValue(getValueTypeID(VTBH), OS);
       OS << '\n';
       return NumBytes + 1;
@@ -850,6 +858,8 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
       }
     } else {
       OS << "OPC_EmitIntegerByHwMode, ";
+      if (!OmitComments)
+        OS << "/*" << VTBH << "*/";
       TypeBytes = EmitVBRValue(getValueTypeID(VTBH), OS);
       OS << ' ';
     }
@@ -905,12 +915,16 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
     } else {
       if (Reg && Reg->EnumValue > 255) {
         OS << "OPC_EmitRegisterByHwMode2, ";
+        if (!OmitComments)
+          OS << "/*" << VTBH << "*/";
         OpBytes = EmitVBRValue(getValueTypeID(VTBH), OS);
         OS << " TARGET_VAL(" << getQualifiedName(Reg->TheDef) << "),\n";
         return OpBytes + 3;
       }
 
       OS << "OPC_EmitRegisterByHwMode, ";
+      if (!OmitComments)
+        OS << "/*" << VTBH << "*/";
       OpBytes = EmitVBRValue(getValueTypeID(VTBH), OS) + 1;
       OS << ' ';
     }
@@ -1080,12 +1094,15 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
       if (!OmitComments)
         OS << "/*#VTs*/";
       OS << ",";
-    }
-    unsigned NumTypeBytes = 0;
+      }
+      unsigned NumTypeBytes = 0;
     if (ByHwMode) {
       for (unsigned i = 0, e = EN->getNumVTs(); i != e; ++i) {
-        OS << " 0,";
-        ++NumTypeBytes;
+        OS << ' ';
+        const ValueTypeByHwMode &VTBH = EN->getVT(i);
+        if (!OmitComments)
+          OS << "/*" << VTBH << "*/";
+        NumTypeBytes += EmitVBRValue(getValueTypeID(VTBH), OS);
       }
     } else {
       for (unsigned i = 0, e = EN->getNumVTs(); i != e; ++i) {
@@ -1346,6 +1363,28 @@ void MatcherTableEmitter::EmitValueTypeFunction(raw_ostream &OS) {
                     /*AddOverride=*/true);
   OS << "{\n";
 
+  OS << "  switch (Index) {\n";
+  OS << "  default: llvm_unreachable(\"Unexpected index\");\n";
+
+  for (const auto &P : ValueTypeMap) {
+    OS << "  case " << (P.second - 1) << ":\n";
+    OS << "    switch (HwMode) {\n";
+    if (!P.first.hasDefault())
+      OS << "    default:\n      return MVT();\n";
+    for (const auto &VT : P.first) {
+      if (VT.first == 0)
+        OS << "    default:\n";
+      else
+        OS << "    case " << VT.first << ":\n";
+      OS << "      return " << getEnumName(VT.second) << ";\n";
+    }
+
+    OS << "    }\n";
+    OS << "    break;\n";
+  }
+
+  OS << "  }\n";
+
   OS << "}\n";
   EndEmitFunction(OS);
 }

From 32006c217aae4539eb8fb78249cac9c3919a8100 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 5 Jan 2026 12:43:39 -0800
Subject: [PATCH 05/16] fixup! Fix missing checks for RegClassByHwMode.

---
 llvm/test/TableGen/RegClassByHwMode.td            | 4 +++-
 llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp | 2 ++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/llvm/test/TableGen/RegClassByHwMode.td b/llvm/test/TableGen/RegClassByHwMode.td
index fa23932a10ef9..25735961a5a70 100644
--- a/llvm/test/TableGen/RegClassByHwMode.td
+++ b/llvm/test/TableGen/RegClassByHwMode.td
@@ -192,7 +192,9 @@ include "Common/RegClassByHwModeCommon.td"
 // ISEL-SDAG-NEXT: OPC_RecordMemRef,
 // ISEL-SDAG-NEXT: OPC_RecordNode, // #0 = 'st' chained node
 // ISEL-SDAG-NEXT: OPC_RecordChild1, // #1 = $val
+// ISEL-SDAG-NEXT: OPC_CheckChild1TypeByHwMode, /*{(*:i64),(m1:i64),(m2:i64)}*/0,
 // ISEL-SDAG-NEXT: OPC_RecordChild2, // #2 = $src
+// ISEL-SDAG-NEXT: OPC_CheckChild2TypeByHwMode, /*{(*:i32),(m3:i64)}*/1,
 // ISEL-SDAG-NEXT: OPC_CheckPredicate0,  // Predicate_unindexedstore
 // ISEL-SDAG-NEXT: OPC_CheckPredicate1,  // Predicate_store
 // ISEL-SDAG-NEXT: OPC_EmitMergeInputChains1_0,
@@ -202,7 +204,7 @@ include "Common/RegClassByHwModeCommon.td"
 // ISEL-SDAG-NEXT: OPC_RecordMemRef,
 // ISEL-SDAG-NEXT: OPC_RecordNode, // #0 = 'ld' chained node
 // ISEL-SDAG-NEXT: OPC_RecordChild1, // #1 = $src
-// ISEL-SDAG-NEXT: OPC_CheckChild1TypeByHwMode, /*{(*:i32),(m3:i64)}*/0,
+// ISEL-SDAG-NEXT: OPC_CheckChild1TypeByHwMode, /*{(*:i32),(m3:i64)}*/1,
 // ISEL-SDAG-NEXT: OPC_CheckPredicate2,  // Predicate_unindexedload
 // ISEL-SDAG-NEXT: OPC_CheckPredicate3,  // Predicate_load
 // ISEL-SDAG-NEXT: OPC_CheckTypeI64,
diff --git a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
index af1b5592e56c9..3ed5fe5d806ae 100644
--- a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
@@ -2368,6 +2368,8 @@ static TypeSetByHwMode getImplicitType(const Record *R, unsigned ResNo,
   }
 
   if (R->isSubClassOf("RegClassByHwMode")) {
+    if (NotRegisters)
+      return TypeSetByHwMode(); // Unknown.
     const CodeGenTarget &T = CDP.getTargetInfo();
     return getTypeForRegClassByHwMode(T, R, TP.getRecord()->getLoc());
   }

From 97e5bb0ef3e9263791354f7c7b7023b2cfb72ff2 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 5 Jan 2026 21:39:50 -0800
Subject: [PATCH 06/16] fixup! clang-format

---
 .../CodeGen/SelectionDAG/SelectionDAGISel.cpp    | 16 ++++++++++------
 llvm/utils/TableGen/DAGISelMatcherEmitter.cpp    |  4 ++--
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 0ff95264af417..e84cb95f3723a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -2755,10 +2755,10 @@ getSimpleVT(const uint8_t *MatcherTable, unsigned &MatcherIndex) {
 /// use GetVBR to decode it.
 LLVM_ATTRIBUTE_ALWAYS_INLINE static MVT
 getHwModeVT(const uint8_t *MatcherTable, unsigned &MatcherIndex,
-                   const SelectionDAGISel &SDISel) {
+            const SelectionDAGISel &SDISel) {
   unsigned Index = MatcherTable[MatcherIndex++];
   if (Index & 128)
-     Index = GetVBR(Index, MatcherTable, MatcherIndex);
+    Index = GetVBR(Index, MatcherTable, MatcherIndex);
 
   return SDISel.getValueTypeByHwMode(Index);
 }
@@ -3168,7 +3168,8 @@ static unsigned IsPredicateKnownToFail(
   }
   case SelectionDAGISel::OPC_CheckTypeByHwMode: {
     MVT VT = getHwModeVT(Table, Index, SDISel);
-    Result = !::CheckType(VT.SimpleTy, N, SDISel.TLI, SDISel.CurDAG->getDataLayout());
+    Result = !::CheckType(VT.SimpleTy, N, SDISel.TLI,
+                          SDISel.CurDAG->getDataLayout());
     return Index;
   }
   case SelectionDAGISel::OPC_CheckTypeRes: {
@@ -3180,7 +3181,8 @@ static unsigned IsPredicateKnownToFail(
   case SelectionDAGISel::OPC_CheckTypeResByHwMode: {
     unsigned Res = Table[Index++];
     MVT VT = getHwModeVT(Table, Index, SDISel);
-    Result = !::CheckType(VT.SimpleTy, N.getValue(Res), SDISel.TLI, SDISel.CurDAG->getDataLayout());
+    Result = !::CheckType(VT.SimpleTy, N.getValue(Res), SDISel.TLI,
+                          SDISel.CurDAG->getDataLayout());
     return Index;
   }
   case SelectionDAGISel::OPC_CheckChild0Type:
@@ -3776,7 +3778,8 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
     case OPC_CheckTypeResByHwMode: {
       unsigned Res = MatcherTable[MatcherIndex++];
       MVT VT = getHwModeVT(MatcherTable, MatcherIndex, *this);
-      if (!::CheckType(VT.SimpleTy, N.getValue(Res), TLI, CurDAG->getDataLayout()))
+      if (!::CheckType(VT.SimpleTy, N.getValue(Res), TLI,
+                       CurDAG->getDataLayout()))
         break;
       continue;
     }
@@ -3896,7 +3899,8 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
     case OPC_CheckChild7TypeByHwMode: {
       MVT VT = getHwModeVT(MatcherTable, MatcherIndex, *this);
       unsigned ChildNo = Opcode - OPC_CheckChild0TypeByHwMode;
-      if (!::CheckChildType(VT.SimpleTy, N, TLI, CurDAG->getDataLayout(), ChildNo))
+      if (!::CheckChildType(VT.SimpleTy, N, TLI, CurDAG->getDataLayout(),
+                            ChildNo))
         break;
       continue;
     }
diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index 17259831634b5..3bea53b13f8e7 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -1094,8 +1094,8 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
       if (!OmitComments)
         OS << "/*#VTs*/";
       OS << ",";
-      }
-      unsigned NumTypeBytes = 0;
+    }
+    unsigned NumTypeBytes = 0;
     if (ByHwMode) {
       for (unsigned i = 0, e = EN->getNumVTs(); i != e; ++i) {
         OS << ' ';

From 2a207fa02c9226d1699b5f49f55e7556ff1e5a88 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 9 Jan 2026 12:07:23 -0800
Subject: [PATCH 07/16] Add emitValueTypeByHwMode.

---
 llvm/utils/TableGen/DAGISelMatcherEmitter.cpp | 38 +++++++++----------
 1 file changed, 17 insertions(+), 21 deletions(-)

diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index 2febdfd254782..72022be4bae88 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -223,6 +223,9 @@ class MatcherTableEmitter {
       Entry = ValueTypeMap.size();
     return Entry - 1;
   }
+
+  unsigned emitValueTypeByHwMode(const ValueTypeByHwMode &VTBH,
+                                 raw_ostream &OS);
 };
 } // end anonymous namespace.
 
@@ -447,6 +450,13 @@ static unsigned emitMVT(MVT VT, raw_ostream &OS) {
   return EmitVBRValue(VT.SimpleTy, OS);
 }
 
+
+unsigned MatcherTableEmitter::emitValueTypeByHwMode(const ValueTypeByHwMode &VTBH,
+                                      raw_ostream &OS) {
+  if (!OmitComments)
+    OS << "/*" << VTBH << "*/";
+  return EmitVBRValue(getValueTypeID(VTBH), OS);
+}
 /// EmitMatcher - Emit bytes for the specified matcher and return
 /// the number of bytes emitted.
 unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
@@ -696,9 +706,7 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
         }
       } else {
         OS << "OPC_CheckTypeByHwMode, ";
-        if (!OmitComments)
-          OS << "/*" << VTBH << "*/";
-        unsigned NumBytes = EmitVBRValue(getValueTypeID(VTBH), OS);
+        unsigned NumBytes = emitValueTypeByHwMode(VTBH, OS);
         OS << '\n';
         return NumBytes + 1;
       }
@@ -713,9 +721,7 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
     }
     OS << "OPC_CheckTypeResByHwMode, " << cast<CheckTypeMatcher>(N)->getResNo()
        << ", ";
-    if (!OmitComments)
-      OS << "/*" << VTBH << "*/";
-    unsigned NumBytes = EmitVBRValue(getValueTypeID(VTBH), OS);
+    unsigned NumBytes = emitValueTypeByHwMode(VTBH, OS);
     OS << '\n';
     return NumBytes + 2;
   }
@@ -740,9 +746,7 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
     } else {
       OS << "OPC_CheckChild" << cast<CheckChildTypeMatcher>(N)->getChildNo()
          << "TypeByHwMode, ";
-      if (!OmitComments)
-        OS << "/*" << VTBH << "*/";
-      unsigned NumBytes = EmitVBRValue(getValueTypeID(VTBH), OS);
+      unsigned NumBytes = emitValueTypeByHwMode(VTBH, OS);
       OS << '\n';
       return NumBytes + 1;
     }
@@ -855,9 +859,7 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
       }
     } else {
       OS << "OPC_EmitIntegerByHwMode, ";
-      if (!OmitComments)
-        OS << "/*" << VTBH << "*/";
-      TypeBytes = EmitVBRValue(getValueTypeID(VTBH), OS);
+      TypeBytes = emitValueTypeByHwMode(VTBH, OS);
       OS << ' ';
     }
     // If the value is 63 or smaller, use the string directly. Otherwise, use
@@ -908,17 +910,13 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
     } else {
       if (Reg && Reg->EnumValue > 255) {
         OS << "OPC_EmitRegisterByHwMode2, ";
-        if (!OmitComments)
-          OS << "/*" << VTBH << "*/";
-        OpBytes = EmitVBRValue(getValueTypeID(VTBH), OS);
+        OpBytes = emitValueTypeByHwMode(VTBH, OS);
         OS << " TARGET_VAL(" << getQualifiedName(Reg->TheDef) << "),\n";
         return OpBytes + 3;
       }
 
       OS << "OPC_EmitRegisterByHwMode, ";
-      if (!OmitComments)
-        OS << "/*" << VTBH << "*/";
-      OpBytes = EmitVBRValue(getValueTypeID(VTBH), OS) + 1;
+      OpBytes = emitValueTypeByHwMode(VTBH, OS) + 1;
       OS << ' ';
     }
     if (Reg) {
@@ -1093,9 +1091,7 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
       for (unsigned i = 0, e = EN->getNumVTs(); i != e; ++i) {
         OS << ' ';
         const ValueTypeByHwMode &VTBH = EN->getVT(i);
-        if (!OmitComments)
-          OS << "/*" << VTBH << "*/";
-        NumTypeBytes += EmitVBRValue(getValueTypeID(VTBH), OS);
+        NumTypeBytes += emitValueTypeByHwMode(VTBH, OS);
       }
     } else {
       for (unsigned i = 0, e = EN->getNumVTs(); i != e; ++i) {

From 3ed711a6947d53873c08481836dfc795e4fed806 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 9 Jan 2026 12:33:09 -0800
Subject: [PATCH 08/16] fixup! clang-format

---
 llvm/utils/TableGen/DAGISelMatcherEmitter.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index 72022be4bae88..d647bb22a1bb9 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -450,9 +450,9 @@ static unsigned emitMVT(MVT VT, raw_ostream &OS) {
   return EmitVBRValue(VT.SimpleTy, OS);
 }
 
-
-unsigned MatcherTableEmitter::emitValueTypeByHwMode(const ValueTypeByHwMode &VTBH,
-                                      raw_ostream &OS) {
+unsigned
+MatcherTableEmitter::emitValueTypeByHwMode(const ValueTypeByHwMode &VTBH,
+                                           raw_ostream &OS) {
   if (!OmitComments)
     OS << "/*" << VTBH << "*/";
   return EmitVBRValue(getValueTypeID(VTBH), OS);

From 31e37383af09cf4972a0111cd0181ce96995f603 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 9 Jan 2026 14:04:58 -0800
Subject: [PATCH 09/16] fixup! Refactor emission of Matcher::CheckType.

---
 llvm/utils/TableGen/DAGISelMatcherEmitter.cpp | 29 ++++++++++---------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index d647bb22a1bb9..1af4dc4e52aac 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -690,9 +690,9 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
 
   case Matcher::CheckType: {
     const ValueTypeByHwMode &VTBH = cast<CheckTypeMatcher>(N)->getType();
-    if (cast<CheckTypeMatcher>(N)->getResNo() == 0) {
-      if (VTBH.isSimple()) {
-        MVT VT = VTBH.getSimple();
+    if (VTBH.isSimple()) {
+      MVT VT = VTBH.getSimple();
+      if (cast<CheckTypeMatcher>(N)->getResNo() == 0) {
         switch (VT.SimpleTy) {
         case MVT::i32:
         case MVT::i64:
@@ -704,14 +704,8 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
           OS << '\n';
           return NumBytes + 1;
         }
-      } else {
-        OS << "OPC_CheckTypeByHwMode, ";
-        unsigned NumBytes = emitValueTypeByHwMode(VTBH, OS);
-        OS << '\n';
-        return NumBytes + 1;
       }
-    }
-    if (VTBH.isSimple()) {
+
       OS << "OPC_CheckTypeRes, " << cast<CheckTypeMatcher>(N)->getResNo()
          << ", ";
       unsigned NumBytes =
@@ -719,11 +713,18 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
       OS << '\n';
       return NumBytes + 2;
     }
-    OS << "OPC_CheckTypeResByHwMode, " << cast<CheckTypeMatcher>(N)->getResNo()
-       << ", ";
-    unsigned NumBytes = emitValueTypeByHwMode(VTBH, OS);
+
+    unsigned OpSize = 1;
+    if (cast<CheckTypeMatcher>(N)->getResNo() == 0) {
+      OS << "OPC_CheckTypeByHwMode, ";
+    } else {
+      OS << "OPC_CheckTypeResByHwMode, " << cast<CheckTypeMatcher>(N)->getResNo()
+         << ", ";
+      OpSize += 1;
+    }
+    OpSize += emitValueTypeByHwMode(VTBH, OS);
     OS << '\n';
-    return NumBytes + 2;
+    return OpSize;
   }
 
   case Matcher::CheckChildType: {

From 8de22bf487465670b118ee2730f2c041cd7dd833 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 9 Jan 2026 15:24:11 -0800
Subject: [PATCH 10/16] Only allow 256 indices for the value type look up
 instead of using a VBR.

---
 llvm/include/llvm/CodeGen/SelectionDAGISel.h       |  2 +-
 llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp |  8 ++------
 llvm/utils/TableGen/DAGISelMatcherEmitter.cpp      | 12 +++++++++---
 3 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
index c96a7ba97201f..4d96ed8ef6e14 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
@@ -466,7 +466,7 @@ class SelectionDAGISel {
     llvm_unreachable("Tblgen should generate this!");
   }
 
-  virtual MVT getValueTypeByHwMode(unsigned Index) const {
+  virtual MVT getValueTypeForHwMode(unsigned Index) const {
     llvm_unreachable("Tblgen should generate the implementation of this!");
   }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index e84cb95f3723a..2af144359686e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -2751,16 +2751,12 @@ getSimpleVT(const uint8_t *MatcherTable, unsigned &MatcherIndex) {
   return static_cast<MVT::SimpleValueType>(SimpleVT);
 }
 
-/// getSimpleVT - Decode a value in MatcherTable, if it's a VBR encoded value,
-/// use GetVBR to decode it.
+/// Decode a HwMode VT in MatcherTable by calling getValueTypeForHwMode.
 LLVM_ATTRIBUTE_ALWAYS_INLINE static MVT
 getHwModeVT(const uint8_t *MatcherTable, unsigned &MatcherIndex,
             const SelectionDAGISel &SDISel) {
   unsigned Index = MatcherTable[MatcherIndex++];
-  if (Index & 128)
-    Index = GetVBR(Index, MatcherTable, MatcherIndex);
-
-  return SDISel.getValueTypeByHwMode(Index);
+  return SDISel.getValueTypeForHwMode(Index);
 }
 
 void SelectionDAGISel::Select_JUMP_TABLE_DEBUG_INFO(SDNode *N) {
diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index 1af4dc4e52aac..3c21252bd8b99 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -219,8 +219,13 @@ class MatcherTableEmitter {
 
   unsigned getValueTypeID(const ValueTypeByHwMode &VT) {
     unsigned &Entry = ValueTypeMap[VT];
-    if (Entry == 0)
+    if (Entry == 0) {
       Entry = ValueTypeMap.size();
+      if (Entry > 256)
+        report_fatal_error(
+            "More ValueType by HwMode than fit in a 8-bit index");
+    }
+
     return Entry - 1;
   }
 
@@ -455,7 +460,8 @@ MatcherTableEmitter::emitValueTypeByHwMode(const ValueTypeByHwMode &VTBH,
                                            raw_ostream &OS) {
   if (!OmitComments)
     OS << "/*" << VTBH << "*/";
-  return EmitVBRValue(getValueTypeID(VTBH), OS);
+  OS << getValueTypeID(VTBH) << ',';
+  return 1;
 }
 /// EmitMatcher - Emit bytes for the specified matcher and return
 /// the number of bytes emitted.
@@ -1347,7 +1353,7 @@ void MatcherTableEmitter::EmitValueTypeFunction(raw_ostream &OS) {
   if (ValueTypeMap.empty())
     return;
 
-  BeginEmitFunction(OS, "MVT", "getValueTypeByHwMode(unsigned Index) const",
+  BeginEmitFunction(OS, "MVT", "getValueTypeForHwMode(unsigned Index) const",
                     /*AddOverride=*/true);
   OS << "{\n";
 

From 4c730312f589daa1bc7da508ee0528982212f269 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 9 Jan 2026 15:27:43 -0800
Subject: [PATCH 11/16] fixup! clang-format

---
 llvm/utils/TableGen/DAGISelMatcherEmitter.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index 3c21252bd8b99..2932e448aba87 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -724,8 +724,8 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N,
     if (cast<CheckTypeMatcher>(N)->getResNo() == 0) {
       OS << "OPC_CheckTypeByHwMode, ";
     } else {
-      OS << "OPC_CheckTypeResByHwMode, " << cast<CheckTypeMatcher>(N)->getResNo()
-         << ", ";
+      OS << "OPC_CheckTypeResByHwMode, "
+         << cast<CheckTypeMatcher>(N)->getResNo() << ", ";
       OpSize += 1;
     }
     OpSize += emitValueTypeByHwMode(VTBH, OS);

From a06c618e357c6bd06d5581ae924abbccccfe6dd3 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 12 Jan 2026 13:30:36 -0800
Subject: [PATCH 12/16] fixup! fix include path after rebase.

---
 llvm/utils/TableGen/DAGISelMatcher.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/utils/TableGen/DAGISelMatcher.h b/llvm/utils/TableGen/DAGISelMatcher.h
index 6c073f48a9cbe..f2a75147c2aca 100644
--- a/llvm/utils/TableGen/DAGISelMatcher.h
+++ b/llvm/utils/TableGen/DAGISelMatcher.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_UTILS_TABLEGEN_COMMON_DAGISELMATCHER_H
 #define LLVM_UTILS_TABLEGEN_COMMON_DAGISELMATCHER_H
 
-#include "InfoByHwMode.h"
+#include "Common/InfoByHwMode.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"

From d9bbb63bdb6b50faec2d7344fed59f2f278c4406 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 13 Jan 2026 10:45:36 -0800
Subject: [PATCH 13/16] fixup! address review comment

---
 llvm/utils/TableGen/DAGISelEmitter.cpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/llvm/utils/TableGen/DAGISelEmitter.cpp b/llvm/utils/TableGen/DAGISelEmitter.cpp
index 5cb5fbcff0109..dc66967584f71 100644
--- a/llvm/utils/TableGen/DAGISelEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelEmitter.cpp
@@ -95,16 +95,16 @@ struct PatternSortingPredicate {
     bool RHSIsFP = false;
 
     if (LT.getNumTypes() != 0) {
-      for (auto VT : LT.getType(0)) {
-        LHSIsVector |= VT.second.isVector();
-        LHSIsFP |= VT.second.isFloatingPoint();
+      for (auto [_, VT] : LT.getType(0)) {
+        LHSIsVector |= VT.isVector();
+        LHSIsFP |= VT.isFloatingPoint();
       }
     }
 
     if (RT.getNumTypes() != 0) {
-      for (auto VT : RT.getType(0)) {
-        RHSIsVector |= VT.second.isVector();
-        RHSIsFP |= VT.second.isFloatingPoint();
+      for (auto [_, VT] : RT.getType(0)) {
+        RHSIsVector |= VT.isVector();
+        RHSIsFP |= VT.isFloatingPoint();
       }
     }
 

From 8c4b460d9beaa71e5b5f54e06bdd1b85ab91261f Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 13 Jan 2026 11:01:50 -0800
Subject: [PATCH 14/16] fixup! Another review comment

---
 llvm/utils/TableGen/DAGISelMatcherEmitter.cpp | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index 5209980d77610..95338b906568a 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -1365,17 +1365,17 @@ void MatcherTableEmitter::EmitValueTypeFunction(raw_ostream &OS) {
   OS << "  switch (Index) {\n";
   OS << "  default: llvm_unreachable(\"Unexpected index\");\n";
 
-  for (const auto &P : ValueTypeMap) {
-    OS << "  case " << (P.second - 1) << ":\n";
+  for (const auto &[VTs, Idx] : ValueTypeMap) {
+    OS << "  case " << (Idx - 1) << ":\n";
     OS << "    switch (HwMode) {\n";
-    if (!P.first.hasDefault())
+    if (!VTs.hasDefault())
       OS << "    default:\n      return MVT();\n";
-    for (const auto &VT : P.first) {
-      if (VT.first == 0)
+    for (const auto [Mode, VT] : VTs) {
+      if (Mode == DefaultMode)
         OS << "    default:\n";
       else
-        OS << "    case " << VT.first << ":\n";
-      OS << "      return " << getEnumName(VT.second) << ";\n";
+        OS << "    case " << Mode << ":\n";
+      OS << "      return " << getEnumName(VT) << ";\n";
     }
 
     OS << "    }\n";

From d3516a04845352317ab505a4c92048880319d404 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 13 Jan 2026 23:33:44 -0800
Subject: [PATCH 15/16] fixup! Address review comments

---
 .../CodeGen/SelectionDAG/SelectionDAGISel.cpp | 127 +++++++-----------
 1 file changed, 50 insertions(+), 77 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index da603eddc6139..2aa775115811a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -3146,8 +3146,9 @@ static unsigned IsPredicateKnownToFail(
     return Index;
   case SelectionDAGISel::OPC_CheckType:
   case SelectionDAGISel::OPC_CheckTypeI32:
-  case SelectionDAGISel::OPC_CheckTypeI64: {
-    MVT::SimpleValueType VT;
+  case SelectionDAGISel::OPC_CheckTypeI64:
+  case SelectionDAGISel::OPC_CheckTypeByHwMode: {
+    MVT VT;
     switch (Opcode) {
     case SelectionDAGISel::OPC_CheckTypeI32:
       VT = MVT::i32;
@@ -3155,28 +3156,23 @@ static unsigned IsPredicateKnownToFail(
     case SelectionDAGISel::OPC_CheckTypeI64:
       VT = MVT::i64;
       break;
+    case SelectionDAGISel::OPC_CheckTypeByHwMode:
+      VT = getHwModeVT(Table, Index, SDISel);
+      break;
     default:
       VT = getSimpleVT(Table, Index);
       break;
     }
-    Result = !::CheckType(VT, N, SDISel.TLI, SDISel.CurDAG->getDataLayout());
-    return Index;
-  }
-  case SelectionDAGISel::OPC_CheckTypeByHwMode: {
-    MVT VT = getHwModeVT(Table, Index, SDISel);
     Result = !::CheckType(VT.SimpleTy, N, SDISel.TLI,
                           SDISel.CurDAG->getDataLayout());
     return Index;
   }
-  case SelectionDAGISel::OPC_CheckTypeRes: {
-    unsigned Res = Table[Index++];
-    Result = !::CheckType(getSimpleVT(Table, Index), N.getValue(Res),
-                          SDISel.TLI, SDISel.CurDAG->getDataLayout());
-    return Index;
-  }
+  case SelectionDAGISel::OPC_CheckTypeRes:
   case SelectionDAGISel::OPC_CheckTypeResByHwMode: {
     unsigned Res = Table[Index++];
-    MVT VT = getHwModeVT(Table, Index, SDISel);
+    MVT VT = Opcode == SelectionDAGISel::OPC_CheckTypeResByHwMode
+                 ? getHwModeVT(Table, Index, SDISel)
+                 : getSimpleVT(Table, Index);
     Result = !::CheckType(VT.SimpleTy, N.getValue(Res), SDISel.TLI,
                           SDISel.CurDAG->getDataLayout());
     return Index;
@@ -3204,8 +3200,16 @@ static unsigned IsPredicateKnownToFail(
   case SelectionDAGISel::OPC_CheckChild4TypeI64:
   case SelectionDAGISel::OPC_CheckChild5TypeI64:
   case SelectionDAGISel::OPC_CheckChild6TypeI64:
-  case SelectionDAGISel::OPC_CheckChild7TypeI64: {
-    MVT::SimpleValueType VT;
+  case SelectionDAGISel::OPC_CheckChild7TypeI64:
+  case SelectionDAGISel::OPC_CheckChild0TypeByHwMode:
+  case SelectionDAGISel::OPC_CheckChild1TypeByHwMode:
+  case SelectionDAGISel::OPC_CheckChild2TypeByHwMode:
+  case SelectionDAGISel::OPC_CheckChild3TypeByHwMode:
+  case SelectionDAGISel::OPC_CheckChild4TypeByHwMode:
+  case SelectionDAGISel::OPC_CheckChild5TypeByHwMode:
+  case SelectionDAGISel::OPC_CheckChild6TypeByHwMode:
+  case SelectionDAGISel::OPC_CheckChild7TypeByHwMode: {
+    MVT VT;
     unsigned ChildNo;
     if (Opcode >= SelectionDAGISel::OPC_CheckChild0TypeI32 &&
         Opcode <= SelectionDAGISel::OPC_CheckChild7TypeI32) {
@@ -3215,24 +3219,14 @@ static unsigned IsPredicateKnownToFail(
                Opcode <= SelectionDAGISel::OPC_CheckChild7TypeI64) {
       VT = MVT::i64;
       ChildNo = Opcode - SelectionDAGISel::OPC_CheckChild0TypeI64;
+    } else if (Opcode >= SelectionDAGISel::OPC_CheckChild0TypeByHwMode &&
+               Opcode <= SelectionDAGISel::OPC_CheckChild7TypeByHwMode) {
+      VT = getHwModeVT(Table, Index, SDISel);
+      ChildNo = Opcode - SelectionDAGISel::OPC_CheckChild0TypeByHwMode;
     } else {
       VT = getSimpleVT(Table, Index);
       ChildNo = Opcode - SelectionDAGISel::OPC_CheckChild0Type;
     }
-    Result = !::CheckChildType(VT, N, SDISel.TLI,
-                               SDISel.CurDAG->getDataLayout(), ChildNo);
-    return Index;
-  }
-  case SelectionDAGISel::OPC_CheckChild0TypeByHwMode:
-  case SelectionDAGISel::OPC_CheckChild1TypeByHwMode:
-  case SelectionDAGISel::OPC_CheckChild2TypeByHwMode:
-  case SelectionDAGISel::OPC_CheckChild3TypeByHwMode:
-  case SelectionDAGISel::OPC_CheckChild4TypeByHwMode:
-  case SelectionDAGISel::OPC_CheckChild5TypeByHwMode:
-  case SelectionDAGISel::OPC_CheckChild6TypeByHwMode:
-  case SelectionDAGISel::OPC_CheckChild7TypeByHwMode: {
-    MVT VT = getHwModeVT(Table, Index, SDISel);
-    unsigned ChildNo = Opcode - SelectionDAGISel::OPC_CheckChild0TypeByHwMode;
     Result = !::CheckChildType(VT.SimpleTy, N, SDISel.TLI,
                                SDISel.CurDAG->getDataLayout(), ChildNo);
     return Index;
@@ -3740,8 +3734,9 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
 
     case OPC_CheckType:
     case OPC_CheckTypeI32:
-    case OPC_CheckTypeI64: {
-      MVT::SimpleValueType VT;
+    case OPC_CheckTypeI64:
+    case OPC_CheckTypeByHwMode: {
+      MVT VT;
       switch (Opcode) {
       case OPC_CheckTypeI32:
         VT = MVT::i32;
@@ -3749,31 +3744,24 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
       case OPC_CheckTypeI64:
         VT = MVT::i64;
         break;
+      case OPC_CheckTypeByHwMode:
+        VT = getHwModeVT(MatcherTable, MatcherIndex, *this);
+        break;
       default:
         VT = getSimpleVT(MatcherTable, MatcherIndex);
         break;
       }
-      if (!::CheckType(VT, N, TLI, CurDAG->getDataLayout()))
-        break;
-      continue;
-    }
-    case OPC_CheckTypeByHwMode: {
-      MVT VT = getHwModeVT(MatcherTable, MatcherIndex, *this);
       if (!::CheckType(VT.SimpleTy, N, TLI, CurDAG->getDataLayout()))
         break;
       continue;
     }
 
-    case OPC_CheckTypeRes: {
-      unsigned Res = MatcherTable[MatcherIndex++];
-      if (!::CheckType(getSimpleVT(MatcherTable, MatcherIndex), N.getValue(Res),
-                       TLI, CurDAG->getDataLayout()))
-        break;
-      continue;
-    }
+    case OPC_CheckTypeRes:
     case OPC_CheckTypeResByHwMode: {
       unsigned Res = MatcherTable[MatcherIndex++];
-      MVT VT = getHwModeVT(MatcherTable, MatcherIndex, *this);
+      MVT VT = Opcode == OPC_CheckTypeResByHwMode
+                   ? getHwModeVT(MatcherTable, MatcherIndex, *this)
+                   : getSimpleVT(MatcherTable, MatcherIndex);
       if (!::CheckType(VT.SimpleTy, N.getValue(Res), TLI,
                        CurDAG->getDataLayout()))
         break;
@@ -3968,8 +3956,9 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
     case OPC_EmitIntegerI8:
     case OPC_EmitIntegerI16:
     case OPC_EmitIntegerI32:
-    case OPC_EmitIntegerI64: {
-      MVT::SimpleValueType VT;
+    case OPC_EmitIntegerI64:
+    case OPC_EmitIntegerByHwMode: {
+      MVT VT;
       switch (Opcode) {
       case OPC_EmitIntegerI8:
         VT = MVT::i8;
@@ -3983,22 +3972,15 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
       case OPC_EmitIntegerI64:
         VT = MVT::i64;
         break;
+      case OPC_EmitIntegerByHwMode:
+        VT = getHwModeVT(MatcherTable, MatcherIndex, *this);
+        break;
       default:
         VT = getSimpleVT(MatcherTable, MatcherIndex);
         break;
       }
       int64_t Val = GetSignedVBR(MatcherTable, MatcherIndex);
       Val = SignExtend64(Val, MVT(VT).getFixedSizeInBits());
-      RecordedNodes.emplace_back(
-          CurDAG->getSignedConstant(Val, SDLoc(NodeToMatch), VT,
-                                    /*isTarget=*/true),
-          nullptr);
-      continue;
-    }
-    case OPC_EmitIntegerByHwMode: {
-      MVT VT = getHwModeVT(MatcherTable, MatcherIndex, *this);
-      int64_t Val = GetSignedVBR(MatcherTable, MatcherIndex);
-      Val = SignExtend64(Val, MVT(VT).getFixedSizeInBits());
       RecordedNodes.emplace_back(
           CurDAG->getSignedConstant(Val, SDLoc(NodeToMatch), VT.SimpleTy,
                                     /*isTarget=*/true),
@@ -4008,8 +3990,9 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
 
     case OPC_EmitRegister:
     case OPC_EmitRegisterI32:
-    case OPC_EmitRegisterI64: {
-      MVT::SimpleValueType VT;
+    case OPC_EmitRegisterI64:
+    case OPC_EmitRegisterByHwMode: {
+      MVT VT;
       switch (Opcode) {
       case OPC_EmitRegisterI32:
         VT = MVT::i32;
@@ -4017,6 +4000,9 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
       case OPC_EmitRegisterI64:
         VT = MVT::i64;
         break;
+      case OPC_EmitRegisterByHwMode:
+        VT = getHwModeVT(MatcherTable, MatcherIndex, *this);
+        break;
       default:
         VT = getSimpleVT(MatcherTable, MatcherIndex);
         break;
@@ -4025,27 +4011,14 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
       RecordedNodes.emplace_back(CurDAG->getRegister(RegNo, VT), nullptr);
       continue;
     }
-    case OPC_EmitRegisterByHwMode: {
-      MVT VT = getHwModeVT(MatcherTable, MatcherIndex, *this);
-      unsigned RegNo = MatcherTable[MatcherIndex++];
-      RecordedNodes.emplace_back(CurDAG->getRegister(RegNo, VT), nullptr);
-      continue;
-    }
-    case OPC_EmitRegister2: {
-      // For targets w/ more than 256 register names, the register enum
-      // values are stored in two bytes in the matcher table (just like
-      // opcodes).
-      MVT::SimpleValueType VT = getSimpleVT(MatcherTable, MatcherIndex);
-      unsigned RegNo = MatcherTable[MatcherIndex++];
-      RegNo |= MatcherTable[MatcherIndex++] << 8;
-      RecordedNodes.emplace_back(CurDAG->getRegister(RegNo, VT), nullptr);
-      continue;
-    }
+    case OPC_EmitRegister2:
     case OPC_EmitRegisterByHwMode2: {
       // For targets w/ more than 256 register names, the register enum
       // values are stored in two bytes in the matcher table (just like
       // opcodes).
-      MVT VT = getHwModeVT(MatcherTable, MatcherIndex, *this);
+      MVT VT = Opcode == OPC_EmitRegisterByHwMode2
+                   ? getHwModeVT(MatcherTable, MatcherIndex, *this)
+                   : getSimpleVT(MatcherTable, MatcherIndex);
       unsigned RegNo = MatcherTable[MatcherIndex++];
       RegNo |= MatcherTable[MatcherIndex++] << 8;
       RecordedNodes.emplace_back(CurDAG->getRegister(RegNo, VT), nullptr);

From c90a5d709329be67e52ffc295432c43fe0f59140 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 14 Jan 2026 18:00:59 -0800
Subject: [PATCH 16/16] fixup! address review comment

---
 llvm/include/llvm/CodeGen/SelectionDAGISel.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
index ea4f1ec37a3f2..8a9e98330f895 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
@@ -77,6 +77,8 @@ class SelectionDAGISel {
   bool MatchFilterFuncName = false;
   StringRef FuncName;
 
+  // HwMode to be used by getValueTypeForHwMode. This will be initialized
+  // based on the subtarget used by the MachineFunction.
   unsigned HwMode;
 
   explicit SelectionDAGISel(TargetMachine &tm,