diff --git a/clang-tools-extra/clang-tidy/misc/ExplicitConstructorCheck.cpp b/clang-tools-extra/clang-tidy/misc/ExplicitConstructorCheck.cpp
index 8c6f8ef978991..23261fe5af61b 100644
--- a/clang-tools-extra/clang-tidy/misc/ExplicitConstructorCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/ExplicitConstructorCheck.cpp
@@ -8,6 +8,7 @@
 
 #include "ExplicitConstructorCheck.h"
 #include "../utils/LexerUtils.h"
+#include "../utils/TypeTraits.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/ASTMatchers/ASTMatchers.h"
@@ -31,27 +32,6 @@ void ExplicitConstructorCheck::registerMatchers(MatchFinder *Finder) {
       this);
 }
 
-static bool declIsStdInitializerList(const NamedDecl *D) {
-  // First use the fast getName() method to avoid unnecessary calls to the
-  // slow getQualifiedNameAsString().
-  return D->getName() == "initializer_list" &&
-         D->getQualifiedNameAsString() == "std::initializer_list";
-}
-
-static bool isStdInitializerList(QualType Type) {
-  Type = Type.getCanonicalType();
-  if (const auto *TS = Type->getAs<TemplateSpecializationType>()) {
-    if (const TemplateDecl *TD = TS->getTemplateName().getAsTemplateDecl())
-      return declIsStdInitializerList(TD);
-  }
-  if (const auto *RT = Type->getAs<RecordType>()) {
-    if (const auto *Specialization =
-            dyn_cast<ClassTemplateSpecializationDecl>(RT->getDecl()))
-      return declIsStdInitializerList(Specialization->getSpecializedTemplate());
-  }
-  return false;
-}
-
 void ExplicitConstructorCheck::check(const MatchFinder::MatchResult &Result) {
   constexpr char NoExpressionWarningMessage[] =
       "%0 must be marked explicit to avoid unintentional implicit conversions";
@@ -79,7 +59,7 @@ void ExplicitConstructorCheck::check(const MatchFinder::MatchResult &Result) {
 
   const ExplicitSpecifier ExplicitSpec = Ctor->getExplicitSpecifier();
 
-  const bool TakesInitializerList = isStdInitializerList(
+  const bool TakesInitializerList = utils::type_traits::isStdInitializerList(
       Ctor->getParamDecl(0)->getType().getNonReferenceType());
   if (ExplicitSpec.isExplicit() &&
       (Ctor->isCopyOrMoveConstructor() || TakesInitializerList)) {
diff --git a/clang-tools-extra/clang-tidy/utils/TypeTraits.cpp b/clang-tools-extra/clang-tidy/utils/TypeTraits.cpp
index c15d90b19d359..136a7b2ed26a8 100644
--- a/clang-tools-extra/clang-tidy/utils/TypeTraits.cpp
+++ b/clang-tools-extra/clang-tidy/utils/TypeTraits.cpp
@@ -9,6 +9,7 @@
 #include "TypeTraits.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclCXX.h"
+#include "clang/AST/DeclTemplate.h"
 #include <optional>
 
 namespace clang::tidy::utils::type_traits {
@@ -146,4 +147,22 @@ bool hasNonTrivialMoveAssignment(QualType Type) {
          Record->hasNonTrivialMoveAssignment();
 }
 
+static bool declIsStdInitializerList(const NamedDecl *D) {
+  return D->isInStdNamespace() && D->getName() == "initializer_list";
+}
+
+bool isStdInitializerList(QualType Type) {
+  Type = Type.getCanonicalType();
+  if (const auto *TS = Type->getAs<TemplateSpecializationType>()) {
+    if (const TemplateDecl *TD = TS->getTemplateName().getAsTemplateDecl())
+      return declIsStdInitializerList(TD);
+  }
+  if (const auto *RT = Type->getAs<RecordType>()) {
+    if (const auto *Specialization =
+            dyn_cast_if_present<ClassTemplateSpecializationDecl>(RT->getDecl()))
+      return declIsStdInitializerList(Specialization->getSpecializedTemplate());
+  }
+  return false;
+}
+
 } // namespace clang::tidy::utils::type_traits
diff --git a/clang-tools-extra/clang-tidy/utils/TypeTraits.h b/clang-tools-extra/clang-tidy/utils/TypeTraits.h
index 98a4a99bf8d4d..03711461b51ac 100644
--- a/clang-tools-extra/clang-tidy/utils/TypeTraits.h
+++ b/clang-tools-extra/clang-tidy/utils/TypeTraits.h
@@ -34,6 +34,9 @@ bool hasNonTrivialMoveConstructor(QualType Type);
 /// Return true if `Type` has a non-trivial move assignment operator.
 bool hasNonTrivialMoveAssignment(QualType Type);
 
+/// Returns `true` if `Type` is a `std::initializer_list<...>` specialization.
+bool isStdInitializerList(QualType Type);
+
 } // namespace clang::tidy::utils::type_traits
 
 #endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_UTILS_TYPETRAITS_H
diff --git a/clang-tools-extra/clangd/CodeCompletionStrings.cpp b/clang-tools-extra/clangd/CodeCompletionStrings.cpp
index 9c4241b54057a..dc86be60a876f 100644
--- a/clang-tools-extra/clangd/CodeCompletionStrings.cpp
+++ b/clang-tools-extra/clangd/CodeCompletionStrings.cpp
@@ -127,11 +127,12 @@ std::string getDeclComment(const ASTContext &Ctx, const NamedDecl &Decl) {
     // not write them into PCH, because they are racy and slow to load.
     assert(!Ctx.getSourceManager().isLoadedSourceLocation(RC->getBeginLoc()));
 
-    comments::FullComment *FC = RC->parse(Ctx, /*PP=*/nullptr, ND);
-    if (!FC)
+    std::string DeclDoc =
+        RC->getFormattedText(Ctx.getSourceManager(), Ctx.getDiagnostics());
+    if (!looksLikeDocComment(DeclDoc))
       return "";
 
-    SymbolDocCommentVisitor V(FC, Ctx.getLangOpts().CommentOpts);
+    SymbolDocCommentVisitor V(DeclDoc, Ctx.getLangOpts().CommentOpts);
     std::string RawDoc;
     llvm::raw_string_ostream OS(RawDoc);
 
diff --git a/clang-tools-extra/clangd/SymbolDocumentation.cpp b/clang-tools-extra/clangd/SymbolDocumentation.cpp
index a50d7a565b1bc..c54f752f04196 100644
--- a/clang-tools-extra/clangd/SymbolDocumentation.cpp
+++ b/clang-tools-extra/clangd/SymbolDocumentation.cpp
@@ -34,9 +34,10 @@ void commandToMarkup(markup::Paragraph &Out, StringRef Command,
                      comments::CommandMarkerKind CommandMarker,
                      StringRef Args) {
   Out.appendBoldText(commandMarkerAsString(CommandMarker) + Command.str());
-  Out.appendSpace();
-  if (!Args.empty())
+  if (!Args.empty()) {
+    Out.appendSpace();
     Out.appendCode(Args.str());
+  }
 }
 
 template <typename T> std::string getArgText(const T *Command) {
@@ -108,6 +109,7 @@ class ParagraphToMarkupDocument
 
       commandToMarkup(Out, C->getCommandName(Traits), C->getCommandMarker(),
                       "");
+      LastChunkEndsWithNewline = false;
     }
   }
 
@@ -159,7 +161,11 @@ class ParagraphToString
     }
   }
 
-  void visitTextComment(const comments::TextComment *C) { Out << C->getText(); }
+  void visitTextComment(const comments::TextComment *C) {
+    Out << C->getText();
+    if (C->hasTrailingNewline())
+      Out << "\n";
+  }
 
   void visitInlineCommandComment(const comments::InlineCommandComment *C) {
     Out << commandMarkerAsString(C->getCommandMarker());
@@ -167,7 +173,6 @@ class ParagraphToString
     std::string ArgText = getArgText(C);
     if (!ArgText.empty())
       Out << " " << ArgText;
-    Out << " ";
   }
 
   void visitHTMLStartTagComment(const comments::HTMLStartTagComment *STC) {
@@ -251,10 +256,7 @@ class BlockCommentToMarkupDocument
       commandToMarkup(P, B->getCommandName(Traits), B->getCommandMarker(),
                       ArgText);
       if (B->getParagraph() && !B->getParagraph()->isWhitespace()) {
-        // For commands with arguments, the paragraph starts after the first
-        // space. Therefore we need to append a space manually in this case.
-        if (!ArgText.empty())
-          P.appendSpace();
+        P.appendSpace();
         ParagraphToMarkupDocument(P, Traits).visit(B->getParagraph());
       }
     }
diff --git a/clang-tools-extra/clangd/SymbolDocumentation.h b/clang-tools-extra/clangd/SymbolDocumentation.h
index 88c7ade633516..95787ae6c92b9 100644
--- a/clang-tools-extra/clangd/SymbolDocumentation.h
+++ b/clang-tools-extra/clangd/SymbolDocumentation.h
@@ -31,17 +31,6 @@ namespace clangd {
 class SymbolDocCommentVisitor
     : public comments::ConstCommentVisitor<SymbolDocCommentVisitor> {
 public:
-  SymbolDocCommentVisitor(comments::FullComment *FC,
-                          const CommentOptions &CommentOpts)
-      : Traits(Allocator, CommentOpts), Allocator() {
-    if (!FC)
-      return;
-
-    for (auto *Block : FC->getBlocks()) {
-      visit(Block);
-    }
-  }
-
   SymbolDocCommentVisitor(llvm::StringRef Documentation,
                           const CommentOptions &CommentOpts)
       : Traits(Allocator, CommentOpts), Allocator() {
diff --git a/clang-tools-extra/clangd/support/Markup.cpp b/clang-tools-extra/clangd/support/Markup.cpp
index 9ba993a04709c..8ecaa823eb280 100644
--- a/clang-tools-extra/clangd/support/Markup.cpp
+++ b/clang-tools-extra/clangd/support/Markup.cpp
@@ -537,16 +537,28 @@ void Paragraph::renderEscapedMarkdown(llvm::raw_ostream &OS) const {
 void Paragraph::renderMarkdown(llvm::raw_ostream &OS) const {
   bool NeedsSpace = false;
   bool HasChunks = false;
+  bool TextEndsWithNewline = false;
   std::string ParagraphText;
   ParagraphText.reserve(EstimatedStringSize);
   llvm::raw_string_ostream ParagraphTextOS(ParagraphText);
   for (auto &C : Chunks) {
+
+    if (TextEndsWithNewline) {
+      ParagraphTextOS << "\n";
+      TextEndsWithNewline = false;
+    }
+
     if (C.SpaceBefore || NeedsSpace)
       ParagraphTextOS << " ";
+
     switch (C.Kind) {
     case ChunkKind::PlainText:
       ParagraphTextOS << renderText(C.Contents, !HasChunks,
                                     /*EscapeMarkdown=*/false);
+      // renderText removes trailing newlines, but in case there are additional
+      // chunks to process, we need to keep track of the trailing newline and
+      // add it in the next iteration.
+      TextEndsWithNewline = llvm::StringRef(C.Contents).ends_with("\n");
       break;
     case ChunkKind::InlineCode:
       ParagraphTextOS << renderInlineBlock(C.Contents);
diff --git a/clang-tools-extra/clangd/unittests/CodeCompletionStringsTests.cpp b/clang-tools-extra/clangd/unittests/CodeCompletionStringsTests.cpp
index de5f533d31645..6b8b003a4fe18 100644
--- a/clang-tools-extra/clangd/unittests/CodeCompletionStringsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/CodeCompletionStringsTests.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "CodeCompletionStrings.h"
+#include "Config.h"
 #include "TestTU.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
 #include "gmock/gmock.h"
@@ -69,6 +70,59 @@ TEST_F(CompletionStringTest, GetDeclCommentBadUTF8) {
             getDeclComment(AST.getASTContext(), findDecl(AST, "X")));
 }
 
+TEST_F(CompletionStringTest, GetDeclCommentForParam) {
+  auto TU =
+      TestTU::withCode("/** @param a this is param a */\nvoid func(int a);");
+  auto AST = TU.build();
+  Config Cfg;
+  Cfg.Documentation.CommentFormat = Config::CommentFormatPolicy::Doxygen;
+  WithContextValue WithCfg(Config::Key, std::move(Cfg));
+  const auto &FD = llvm::cast<FunctionDecl>(findDecl(AST, "func"));
+  EXPECT_EQ(FD.getNumParams(), 1UL);
+  EXPECT_EQ("this is param a",
+            getDeclComment(AST.getASTContext(), *FD.parameters()[0]));
+}
+
+TEST_F(CompletionStringTest, GetDeclCommentForMultipleParams) {
+  auto TU = TestTU::withCode("/** @param a this is param a\n * @param b this "
+                             "is param b\n */\nvoid func(int a, int b);");
+  auto AST = TU.build();
+  Config Cfg;
+  Cfg.Documentation.CommentFormat = Config::CommentFormatPolicy::Doxygen;
+  WithContextValue WithCfg(Config::Key, std::move(Cfg));
+  const auto &FD = llvm::cast<FunctionDecl>(findDecl(AST, "func"));
+  EXPECT_EQ(FD.getNumParams(), 2UL);
+  EXPECT_EQ("this is param a",
+            getDeclComment(AST.getASTContext(), *FD.parameters()[0]));
+  EXPECT_EQ("this is param b",
+            getDeclComment(AST.getASTContext(), *FD.parameters()[1]));
+}
+
+TEST_F(CompletionStringTest, GetDeclCommentForUndocumentedParam) {
+  auto TU =
+      TestTU::withCode("/** @param c this is param c */\nvoid func(int a);");
+  auto AST = TU.build();
+  Config Cfg;
+  Cfg.Documentation.CommentFormat = Config::CommentFormatPolicy::Doxygen;
+  WithContextValue WithCfg(Config::Key, std::move(Cfg));
+  const auto &FD = llvm::cast<FunctionDecl>(findDecl(AST, "func"));
+  EXPECT_EQ(FD.getNumParams(), 1UL);
+  EXPECT_EQ("", getDeclComment(AST.getASTContext(), *FD.parameters()[0]));
+}
+
+TEST_F(CompletionStringTest, GetDeclCommentForParamWithUnknownDoxygenCommand) {
+  auto TU = TestTU::withCode("/** @param a this is param a and an\n * @unknown "
+                             "command\n */\nvoid func(int a);");
+  auto AST = TU.build();
+  Config Cfg;
+  Cfg.Documentation.CommentFormat = Config::CommentFormatPolicy::Doxygen;
+  WithContextValue WithCfg(Config::Key, std::move(Cfg));
+  const auto &FD = llvm::cast<FunctionDecl>(findDecl(AST, "func"));
+  EXPECT_EQ(FD.getNumParams(), 1UL);
+  EXPECT_EQ("this is param a and an\n@unknown command",
+            getDeclComment(AST.getASTContext(), *FD.parameters()[0]));
+}
+
 TEST_F(CompletionStringTest, MultipleAnnotations) {
   Builder.AddAnnotation("Ano1");
   Builder.AddAnnotation("Ano2");
diff --git a/clang-tools-extra/clangd/unittests/HoverTests.cpp b/clang-tools-extra/clangd/unittests/HoverTests.cpp
index 7b168b0bdca60..476494cdd3e6d 100644
--- a/clang-tools-extra/clangd/unittests/HoverTests.cpp
+++ b/clang-tools-extra/clangd/unittests/HoverTests.cpp
@@ -4431,6 +4431,56 @@ brief doc
 ### Details
 
 longer doc)"},
+      {[](HoverInfo &HI) {
+         HI.Kind = index::SymbolKind::Function;
+         HI.Documentation = "@brief brief doc\n"
+                            "@unknown command is treated as an inline command";
+         HI.Definition = "int foo(int a)";
+         HI.ReturnType = "int";
+         HI.Name = "foo";
+         HI.Parameters.emplace();
+         HI.Parameters->emplace_back();
+         HI.Parameters->back().Type = "int";
+         HI.Parameters->back().Name = "a";
+       },
+       R"(### function `foo`
+
+---
+→ `int`
+
+Parameters:
+
+- `int a`
+
+@brief brief doc  
+@unknown command is treated as an inline command
+
+---
+```cpp
+int foo(int a)
+```)",
+       R"(### function
+
+---
+```cpp
+int foo(int a)
+```
+
+---
+### Brief
+
+brief doc
+**@unknown** command is treated as an inline command
+
+---
+### Parameters
+
+- `int a`
+
+---
+### Returns
+
+`int`)"},
   };
 
   for (const auto &C : Cases) {
@@ -5203,6 +5253,22 @@ TEST(Hover, FunctionParameters) {
        "### param\n\n---\n```cpp\n// In foo\nint b\n```\n\n---\nthis is "
        "\\<b>doc\\</b> \\<html-tag attribute/> \\<another-html-tag "
        "attribute=\"value\">for\\</another-html-tag> `b`\n\n---\nType: `int`"},
+      {R"cpp(/// Function doc
+      /// @param a the next command is an
+      /// @unknown command.
+      void foo(int [[^a]]);
+    )cpp",
+       [](HoverInfo &HI) {
+         HI.Name = "a";
+         HI.Kind = index::SymbolKind::Parameter;
+         HI.NamespaceScope = "";
+         HI.LocalScope = "foo::";
+         HI.Type = "int";
+         HI.Definition = "int a";
+         HI.Documentation = "the next command is an\n @unknown command.";
+       },
+       "### param\n\n---\n```cpp\n// In foo\nint a\n```\n\n---\nthe next "
+       "command is an\n**@unknown** command.\n\n---\nType: `int`"},
   };
 
   // Create a tiny index, so tests above can verify documentation is fetched.
diff --git a/clang-tools-extra/clangd/unittests/SymbolDocumentationTests.cpp b/clang-tools-extra/clangd/unittests/SymbolDocumentationTests.cpp
index 676f7dfc74483..d140ab00a175f 100644
--- a/clang-tools-extra/clangd/unittests/SymbolDocumentationTests.cpp
+++ b/clang-tools-extra/clangd/unittests/SymbolDocumentationTests.cpp
@@ -10,6 +10,7 @@
 #include "support/Markup.h"
 #include "clang/Basic/CommentOptions.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/raw_ostream.h"
 #include "gtest/gtest.h"
 
 namespace clang {
@@ -732,5 +733,104 @@ line
   }
 }
 
+TEST(SymbolDocumentation, ParameterDocToString) {
+  CommentOptions CommentOpts;
+
+  struct Case {
+    llvm::StringRef Documentation;
+    llvm::StringRef ExpectedOutputString;
+    llvm::StringRef ParameterName;
+  } Cases[] = {
+      {"This documentation does not contain parameter docs", "", "a"},
+      {"@param a this is a parameter", "", "not_exists"},
+      {"@param a this is a parameter", " this is a parameter", "a"},
+      {R"(@param a parameter doc with an \p inline command)",
+       R"( parameter doc with an \p inline command)", "a"},
+      {R"(@param a parameter doc with an \unknown command)",
+       R"( parameter doc with an \unknown command)", "a"},
+      {"@param a parameter doc with an @unknown command",
+       " parameter doc with an @unknown command", "a"},
+      {R"(@param a parameter doc with
+multiple lines)",
+       R"( parameter doc with
+multiple lines)",
+       "a"},
+      {R"(@param a parameter doc with an
+@unknown command starting a new line)",
+       R"( parameter doc with an
+@unknown command starting a new line)",
+       "a"},
+      {R"(@param a parameter doc with a
+@note command which is a new block command and therefore ends the parameter doc paragraph)",
+       R"( parameter doc with a
+)",
+       "a"},
+      {R"(Unrelated docs
+@param a parameter doc
+
+New paragraph with unrelated docs)",
+       " parameter doc", "a"},
+  };
+  for (const auto &C : Cases) {
+    std::string Result;
+    llvm::raw_string_ostream OS(Result);
+    SymbolDocCommentVisitor SymbolDoc(C.Documentation, CommentOpts);
+
+    SymbolDoc.parameterDocToString(C.ParameterName, OS);
+
+    EXPECT_EQ(Result, C.ExpectedOutputString);
+  }
+}
+
+TEST(SymbolDocumentation, TemplateParameterDocToString) {
+  CommentOptions CommentOpts;
+
+  struct Case {
+    llvm::StringRef Documentation;
+    llvm::StringRef ExpectedOutputString;
+    llvm::StringRef TemplateParameterName;
+  } Cases[] = {
+      {"This documentation does not contain parameter docs", "", "a"},
+      {"@tparam a this is a template type parameter", "", "not_exists"},
+      {"@tparam a this is a template type parameter",
+       " this is a template type parameter", "a"},
+      {R"(@tparam a template type parameter doc with an \p inline command)",
+       R"( template type parameter doc with an \p inline command)", "a"},
+      {R"(@tparam a template type parameter doc with an \unknown command)",
+       R"( template type parameter doc with an \unknown command)", "a"},
+      {"@tparam a template type parameter doc with an @unknown command",
+       " template type parameter doc with an @unknown command", "a"},
+      {R"(@tparam a template type parameter doc with
+multiple lines)",
+       R"( template type parameter doc with
+multiple lines)",
+       "a"},
+      {R"(@tparam a template type parameter doc with an
+@unknown command starting a new line)",
+       R"( template type parameter doc with an
+@unknown command starting a new line)",
+       "a"},
+      {R"(@tparam a template type parameter doc with a
+@note command which is a new block command and therefore ends the template type parameter doc paragraph)",
+       R"( template type parameter doc with a
+)",
+       "a"},
+      {R"(Unrelated docs
+@tparam a template type parameter doc
+
+New paragraph with unrelated docs)",
+       " template type parameter doc", "a"},
+  };
+  for (const auto &C : Cases) {
+    std::string Result;
+    llvm::raw_string_ostream OS(Result);
+    SymbolDocCommentVisitor SymbolDoc(C.Documentation, CommentOpts);
+
+    SymbolDoc.templateTypeParmDocToString(C.TemplateParameterName, OS);
+
+    EXPECT_EQ(Result, C.ExpectedOutputString);
+  }
+}
+
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/unittests/clang-tidy/CMakeLists.txt b/clang-tools-extra/unittests/clang-tidy/CMakeLists.txt
index 26ceb977d27a6..00498813d1e52 100644
--- a/clang-tools-extra/unittests/clang-tidy/CMakeLists.txt
+++ b/clang-tools-extra/unittests/clang-tidy/CMakeLists.txt
@@ -35,6 +35,7 @@ add_extra_unittest(ClangTidyTests
   UsingInserterTest.cpp
   ReadabilityModuleTest.cpp
   TransformerClangTidyCheckTest.cpp
+  TypeTraitsTest.cpp
   )
 
 clang_target_link_libraries(ClangTidyTests
diff --git a/clang-tools-extra/unittests/clang-tidy/TypeTraitsTest.cpp b/clang-tools-extra/unittests/clang-tidy/TypeTraitsTest.cpp
new file mode 100644
index 0000000000000..13261c3aa9e52
--- /dev/null
+++ b/clang-tools-extra/unittests/clang-tidy/TypeTraitsTest.cpp
@@ -0,0 +1,60 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../clang-tidy/utils/TypeTraits.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/Tooling/Tooling.h"
+#include "gtest/gtest.h"
+
+namespace clang::tidy::test {
+namespace {
+
+using namespace ast_matchers;
+
+// Returns whether the type of the declaration named `DeclName` in `Code` is a
+// std::initializer_list specialization.
+bool declTypeIsStdInitializerList(StringRef Code, StringRef DeclName) {
+  std::unique_ptr<ASTUnit> AST =
+      tooling::buildASTFromCodeWithArgs(Code, {"-std=c++17"});
+  EXPECT_NE(AST, nullptr);
+  auto Matches =
+      match(valueDecl(hasName(DeclName)).bind("d"), AST->getASTContext());
+  EXPECT_EQ(Matches.size(), 1u);
+  const auto *D = Matches[0].getNodeAs<ValueDecl>("d");
+  return utils::type_traits::isStdInitializerList(D->getType());
+}
+
+constexpr char InitializerListDecl[] =
+    "namespace std { template <typename T> class initializer_list {}; }\n";
+
+TEST(IsStdInitializerListTest, MatchesSpecialization) {
+  EXPECT_TRUE(declTypeIsStdInitializerList(
+      std::string(InitializerListDecl) + "std::initializer_list<int> v;", "v"));
+}
+
+TEST(IsStdInitializerListTest, MatchesDependentSpecialization) {
+  EXPECT_TRUE(declTypeIsStdInitializerList(
+      std::string(InitializerListDecl) +
+          "template <typename T> void f(std::initializer_list<T> p);",
+      "p"));
+}
+
+TEST(IsStdInitializerListTest, RejectsBuiltin) {
+  EXPECT_FALSE(declTypeIsStdInitializerList("int v;", "v"));
+}
+
+TEST(IsStdInitializerListTest, RejectsNonStdInitializerList) {
+  EXPECT_FALSE(declTypeIsStdInitializerList(
+      "template <typename T> class initializer_list {}; "
+      "initializer_list<int> v;",
+      "v"));
+}
+
+} // namespace
+} // namespace clang::tidy::test
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 09d56e0640b89..a9196f0f877c5 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -523,7 +523,11 @@ Attribute Changes in Clang
   about pointer lifetimes. It may be used to power optimizations in the future,
   however there are no concrete plans to do so at the moment.
 
-* The ``modular_format`` attribute now supports the ``fixed`` aspect for C
+- The attributes ``[[clang::opencl_global_device]]`` and ``[[clang::opencl_global_host]]``
+  are now deprecated. Clang emits a ``-Wdeprecated-attributes`` warning when
+  they are used.
+  
+- The ``modular_format`` attribute now supports the ``fixed`` aspect for C
   ISO 18037 fixed-point ``printf`` specifiers.
 
 Improvements to Clang's diagnostics
diff --git a/clang/include/clang/AST/Comment.h b/clang/include/clang/AST/Comment.h
index 9ea86089373d5..84e5444675ae1 100644
--- a/clang/include/clang/AST/Comment.h
+++ b/clang/include/clang/AST/Comment.h
@@ -344,15 +344,6 @@ class InlineCommandComment : public InlineContentComment {
   ArrayRef<Argument> Args;
 
 public:
-  InlineCommandComment(SourceLocation LocBegin, SourceLocation LocEnd,
-                       unsigned CommandID, InlineCommandRenderKind RK,
-                       ArrayRef<Argument> Args)
-      : InlineContentComment(CommentKind::InlineCommandComment, LocBegin,
-                             LocEnd),
-        Args(Args) {
-    InlineCommandCommentBits.RenderKind = llvm::to_underlying(RK);
-    InlineCommandCommentBits.CommandID = CommandID;
-  }
   InlineCommandComment(SourceLocation LocBegin, SourceLocation LocEnd,
                        unsigned CommandID, InlineCommandRenderKind RK,
                        CommandMarkerKind CommandMarker, ArrayRef<Argument> Args)
diff --git a/clang/include/clang/AST/CommentLexer.h b/clang/include/clang/AST/CommentLexer.h
index 9aa1681cb2c5c..194a31cb7b934 100644
--- a/clang/include/clang/AST/CommentLexer.h
+++ b/clang/include/clang/AST/CommentLexer.h
@@ -33,9 +33,12 @@ enum TokenKind {
   eof,
   newline,
   text,
-  unknown_command,   // Command that does not have an ID.
-  backslash_command, // Command with an ID, that used backslash marker.
-  at_command,        // Command with an ID, that used 'at' marker.
+  unknown_backslash_command, // Command that does not have an ID, that used
+                             // backslash marker.
+  unknown_at_command,        // Command that does not have an ID, that used 'at'
+                             // marker.
+  backslash_command,         // Command with an ID, that used backslash marker.
+  at_command,                // Command with an ID, that used 'at' marker.
   verbatim_block_begin,
   verbatim_block_line,
   verbatim_block_end,
@@ -107,12 +110,12 @@ class Token {
   }
 
   StringRef getUnknownCommandName() const LLVM_READONLY {
-    assert(is(tok::unknown_command));
+    assert(is(tok::unknown_backslash_command) || is(tok::unknown_at_command));
     return StringRef(TextPtr, IntVal);
   }
 
   void setUnknownCommandName(StringRef Name) {
-    assert(is(tok::unknown_command));
+    assert(is(tok::unknown_backslash_command) || is(tok::unknown_at_command));
     TextPtr = Name.data();
     IntVal = Name.size();
   }
diff --git a/clang/include/clang/AST/CommentSema.h b/clang/include/clang/AST/CommentSema.h
index 8dc6e50763dc5..3a7eda6538cbe 100644
--- a/clang/include/clang/AST/CommentSema.h
+++ b/clang/include/clang/AST/CommentSema.h
@@ -136,11 +136,13 @@ class Sema {
 
   InlineContentComment *actOnUnknownCommand(SourceLocation LocBegin,
                                             SourceLocation LocEnd,
-                                            StringRef CommandName);
+                                            StringRef CommandName,
+                                            CommandMarkerKind CommandMarker);
 
   InlineContentComment *actOnUnknownCommand(SourceLocation LocBegin,
                                             SourceLocation LocEnd,
-                                            unsigned CommandID);
+                                            unsigned CommandID,
+                                            CommandMarkerKind CommandMarker);
 
   TextComment *actOnText(SourceLocation LocBegin,
                          SourceLocation LocEnd,
diff --git a/clang/include/clang/Analysis/Analyses/LifetimeSafety/Checker.h b/clang/include/clang/Analysis/Analyses/LifetimeSafety/Checker.h
index 1ad3f9da61f60..99ad816311e74 100644
--- a/clang/include/clang/Analysis/Analyses/LifetimeSafety/Checker.h
+++ b/clang/include/clang/Analysis/Analyses/LifetimeSafety/Checker.h
@@ -29,7 +29,8 @@ void runLifetimeChecker(const LoanPropagationAnalysis &LoanPropagation,
                         const MovedLoansAnalysis &MovedLoans,
                         const LiveOriginsAnalysis &LiveOrigins,
                         FactManager &FactMgr, AnalysisDeclContext &ADC,
-                        LifetimeSafetySemaHelper *SemaHelper);
+                        LifetimeSafetySemaHelper *SemaHelper,
+                        const LifetimeSafetyOpts &LSOpts);
 
 } // namespace clang::lifetimes::internal
 
diff --git a/clang/include/clang/Analysis/Analyses/LifetimeSafety/LifetimeSafety.h b/clang/include/clang/Analysis/Analyses/LifetimeSafety/LifetimeSafety.h
index 28886b826f72f..80a23f18baebd 100644
--- a/clang/include/clang/Analysis/Analyses/LifetimeSafety/LifetimeSafety.h
+++ b/clang/include/clang/Analysis/Analyses/LifetimeSafety/LifetimeSafety.h
@@ -38,6 +38,9 @@ struct LifetimeSafetyOpts {
   /// Maximum number of CFG blocks to analyze. Functions with larger CFGs will
   /// be skipped.
   size_t MaxCFGBlocks;
+
+  /// Whether to suggest lifetime annotations.
+  bool SuggestAnnotations;
 };
 
 /// Enum to track functions visible across or within TU.
@@ -157,6 +160,7 @@ class LifetimeSafetySemaHelper {
 /// The main entry point for the analysis.
 void runLifetimeSafetyAnalysis(AnalysisDeclContext &AC,
                                LifetimeSafetySemaHelper *SemaHelper,
+                               const LifetimeSafetyOpts &Opts,
                                LifetimeSafetyStats &Stats, bool CollectStats);
 
 namespace internal {
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 7c1c88241aaa8..d806adb4be4b8 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -5214,6 +5214,8 @@ As ``global_device`` and ``global_host`` are a subset of
 ``global_device`` and ``global_host`` address spaces to
 ``__global/opencl_global`` address spaces (following ISO/IEC TR 18037 5.1.3
 "Address space nesting and rules for pointers").
+
+These attributes are deprecated and may be removed in a future version of Clang.
   }];
 }
 
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 34ea29f153474..dc7842846e5da 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11878,6 +11878,9 @@ def err_opencl_type_not_found : Error<
   "%0 type %1 not found; include the base header with -finclude-default-header">;
 def warn_opencl_attr_deprecated_ignored : Warning <
   "%0 attribute is deprecated and ignored in %1">, InGroup<IgnoredAttributes>;
+def warn_deprecated_attribute : Warning<
+  "'%0' attribute is deprecated and may be removed in a future version of Clang">,
+  InGroup<DeprecatedAttributes>;
 def err_opencl_variadic_function : Error<
   "invalid prototype, variadic arguments are not allowed in OpenCL">;
 def err_opencl_requires_extension : Error<
diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
index cadb040048384..20b110b38ff78 100644
--- a/clang/lib/AST/ByteCode/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -5398,10 +5398,11 @@ bool Compiler<Emitter>::visitDtorCall(const VarDecl *VD, const APValue &Value) {
   DeclScope<Emitter> LocalScope(this, VD);
   // Create a local variable to use as the instance.
   QualType Ty = VD->getType();
-  Descriptor *D = P.createDescriptor(
-      VD, Ty.getTypePtr(), Descriptor::InlineDescMD, /*IsConst=*/false,
-      /*IsTemporary=*/false, /*IsMutable=*/false,
-      /*IsVolatile=*/Ty.isVolatileQualified(), nullptr);
+  Descriptor *D =
+      P.createDescriptor(VD, Ty.getTypePtr(), Descriptor::InlineDescMD,
+                         /*IsConst=*/Ty.isConstQualified(),
+                         /*IsTemporary=*/false, /*IsMutable=*/false,
+                         /*IsVolatile=*/Ty.isVolatileQualified(), nullptr);
   if (!D)
     return false;
 
@@ -7671,16 +7672,21 @@ bool Compiler<Emitter>::VisitVectorUnaryOperator(const UnaryOperator *E) {
 
 template <class Emitter>
 bool Compiler<Emitter>::visitDeclRef(const ValueDecl *D, const Expr *E) {
-  if (DiscardResult)
-    return true;
-
-  if (const auto *ECD = dyn_cast<EnumConstantDecl>(D))
+  if (const auto *ECD = dyn_cast<EnumConstantDecl>(D)) {
+    if (DiscardResult)
+      return true;
     return this->emitConst(ECD->getInitVal(), E);
+  }
   if (const auto *FuncDecl = dyn_cast<FunctionDecl>(D)) {
+    if (DiscardResult)
+      return true;
     const Function *F = getFunction(FuncDecl);
     return F && this->emitGetFnPtr(F, E);
   }
   if (const auto *TPOD = dyn_cast<TemplateParamObjectDecl>(D)) {
+    if (DiscardResult)
+      return true;
+
     if (UnsignedOrNone Index = P.getOrCreateGlobal(D)) {
       if (OptPrimType T = classify(D->getType())) {
         if (!this->visitAPValue(TPOD->getValue(), *T, E))
@@ -7704,10 +7710,19 @@ bool Compiler<Emitter>::visitDeclRef(const ValueDecl *D, const Expr *E) {
   QualType DeclType = D->getType();
   bool IsReference = DeclType->isReferenceType();
 
+  auto maybePopPtr = [&]() -> bool {
+    if (DiscardResult)
+      return this->emitPopPtr(E);
+    return true;
+  };
+
   // Function parameters.
   // Note that it's important to check them first since we might have a local
   // variable created for a ParmVarDecl as well.
   if (const auto *PVD = dyn_cast<ParmVarDecl>(D)) {
+    if (DiscardResult)
+      return true;
+
     if (Ctx.getLangOpts().CPlusPlus && !Ctx.getLangOpts().CPlusPlus11 &&
         !DeclType->isIntegralOrEnumerationType()) {
       return this->emitInvalidDeclRef(cast<DeclRefExpr>(E),
@@ -7729,9 +7744,9 @@ bool Compiler<Emitter>::visitDeclRef(const ValueDecl *D, const Expr *E) {
     const unsigned Offset = It->second.Offset;
     if (IsReference) {
       assert(classifyPrim(E) == PT_Ptr);
-      return this->emitGetRefLocal(Offset, E);
+      return this->emitGetRefLocal(Offset, E) && maybePopPtr();
     }
-    return this->emitGetPtrLocal(Offset, E);
+    return this->emitGetPtrLocal(Offset, E) && maybePopPtr();
   }
   // Global variables.
   if (auto GlobalIndex = P.getGlobal(D)) {
@@ -7740,10 +7755,11 @@ bool Compiler<Emitter>::visitDeclRef(const ValueDecl *D, const Expr *E) {
         return this->emitGetGlobal(classifyPrim(E), *GlobalIndex, E);
       if (!Ctx.getLangOpts().CPlusPlus23)
         return this->emitGetGlobalUnchecked(classifyPrim(E), *GlobalIndex, E);
-      return this->emitGetRefGlobal(*GlobalIndex, E);
+
+      return this->emitGetRefGlobal(*GlobalIndex, E) && maybePopPtr();
     }
 
-    return this->emitGetPtrGlobal(*GlobalIndex, E);
+    return this->emitGetPtrGlobal(*GlobalIndex, E) && maybePopPtr();
   }
 
   // In case we need to re-visit a declaration.
@@ -7773,8 +7789,8 @@ bool Compiler<Emitter>::visitDeclRef(const ValueDecl *D, const Expr *E) {
       auto [Offset, IsPtr] = It->second;
 
       if (IsPtr)
-        return this->emitGetThisFieldPtr(Offset, E);
-      return this->emitGetPtrThisField(Offset, E);
+        return this->emitGetThisFieldPtr(Offset, E) && maybePopPtr();
+      return this->emitGetPtrThisField(Offset, E) && maybePopPtr();
     }
   }
 
@@ -7785,11 +7801,14 @@ bool Compiler<Emitter>::visitDeclRef(const ValueDecl *D, const Expr *E) {
   }
 
   if (const auto *BD = dyn_cast<BindingDecl>(D))
-    return this->visit(BD->getBinding());
+    return this->delegate(BD->getBinding());
 
   // Avoid infinite recursion.
-  if (D == InitializingDecl)
+  if (D == InitializingDecl) {
+    if (DiscardResult)
+      return true;
     return this->emitDummyPtr(D, E);
+  }
 
   // Try to lazily visit (or emit dummy pointers for) declarations
   // we haven't seen yet.
@@ -7803,6 +7822,9 @@ bool Compiler<Emitter>::visitDeclRef(const ValueDecl *D, const Expr *E) {
         DeclType.isConstant(Ctx.getASTContext()) && !VD->isWeak() &&
         VD->evaluateValue())
       return revisit(VD, /*IsConstexprUnknown=*/false);
+
+    if (DiscardResult)
+      return true;
     return this->emitDummyPtr(D, E);
   }
 
@@ -7848,6 +7870,8 @@ bool Compiler<Emitter>::visitDeclRef(const ValueDecl *D, const Expr *E) {
                                       /*InitializerFailed=*/true, E);
   }
 
+  if (DiscardResult)
+    return true;
   return this->emitDummyPtr(
       D, E, Ctx.getLangOpts().CPlusPlus23 && DeclType->isReferenceType());
 }
diff --git a/clang/lib/AST/ByteCode/Context.cpp b/clang/lib/AST/ByteCode/Context.cpp
index 4beb35a9a7b43..40cf29efcfb4f 100644
--- a/clang/lib/AST/ByteCode/Context.cpp
+++ b/clang/lib/AST/ByteCode/Context.cpp
@@ -174,6 +174,8 @@ bool Context::evaluateDestruction(State &Parent, const VarDecl *VD,
     return false;
   }
 
+  assert(Stk.empty());
+
   return true;
 }
 
diff --git a/clang/lib/AST/ByteCode/EvalEmitter.cpp b/clang/lib/AST/ByteCode/EvalEmitter.cpp
index 7aeac16b9639e..83dac57353944 100644
--- a/clang/lib/AST/ByteCode/EvalEmitter.cpp
+++ b/clang/lib/AST/ByteCode/EvalEmitter.cpp
@@ -77,11 +77,14 @@ EvaluationResult EvalEmitter::interpretDestructor(const VarDecl *VD,
                                                   const APValue &Value) {
   assert(VD);
   S.setEvalLocation(VD->getLocation());
+  S.EvaluatingDecl = VD;
+  S.EvalKind = EvaluationKind::Dtor;
   EvalResult.setSource(VD);
 
   if (!this->visitDtorCall(VD, Value))
     EvalResult.setInvalid();
 
+  S.EvaluatingDecl = nullptr;
   return std::move(this->EvalResult);
 }
 
@@ -278,7 +281,17 @@ template <> bool EvalEmitter::emitRet<PT_Ptr>(SourceInfo Info) {
     if (!Ptr.isLive() && !Ptr.isTemporary())
       return false;
 
-    EvalResult.takeValue(Ptr.toAPValue(Ctx.getASTContext()));
+    // If the variable of this pointer is being evaluated when returning
+    // its value, mark it as constexpr-unknown.
+    APValue V = Ptr.toAPValue(Ctx.getASTContext());
+    if (const Descriptor *DeclDesc = Ptr.getDeclDesc();
+        DeclDesc && S.EvaluatingDecl &&
+        DeclDesc->asVarDecl() == S.EvaluatingDecl &&
+        S.getLangOpts().CPlusPlus23 &&
+        S.EvaluatingDecl->getType()->isReferenceType()) {
+      V.setConstexprUnknown(true);
+    }
+    EvalResult.takeValue(std::move(V));
   }
 
   return true;
diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
index 3d5fda7ddf3c7..71021815baeef 100644
--- a/clang/lib/AST/ByteCode/Interp.cpp
+++ b/clang/lib/AST/ByteCode/Interp.cpp
@@ -205,14 +205,16 @@ static bool CheckTemporary(InterpState &S, CodePtr OpPC, const Block *B,
     // FIXME(perf): Since we do this check on every Load from a static
     // temporary, it might make sense to cache the value of the
     // isUsableInConstantExpressions call.
-    if (B->getEvalID() != S.EvalID &&
-        !MTE->isUsableInConstantExpressions(S.getASTContext())) {
+    if (S.checkingConstantDestruction() ||
+        (B->getEvalID() != S.EvalID &&
+         !MTE->isUsableInConstantExpressions(S.getASTContext()))) {
       const SourceInfo &E = S.Current->getSource(OpPC);
       S.FFDiag(E, diag::note_constexpr_access_static_temporary, 1) << AK;
       noteValueLocation(S, B);
       return false;
     }
   }
+
   return true;
 }
 
@@ -459,7 +461,12 @@ bool CheckConstant(InterpState &S, CodePtr OpPC, const Descriptor *Desc,
   assert(Desc);
 
   const auto *D = Desc->asVarDecl();
-  if (!D || D == S.EvaluatingDecl || D->isConstexpr())
+  if (S.checkingConstantDestruction(D)) {
+    // If we're checking for a constant destructor for this variable, we can
+    // only read from it if it is constant.
+    if (D->getType().isConstQualified())
+      return true;
+  } else if (!D || D == S.EvaluatingDecl || D->isConstexpr())
     return true;
 
   // If we're evaluating the initializer for a constexpr variable in C23, we may
@@ -509,6 +516,9 @@ bool CheckConstant(InterpState &S, CodePtr OpPC, const Descriptor *Desc,
 
 static bool CheckConstant(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
                           AccessKinds AK = AK_Read) {
+  if (S.checkingConstantDestruction(Ptr))
+    return CheckConstant(S, OpPC, Ptr.getDeclDesc(), AK);
+
   if (!Ptr.isStatic() || !Ptr.isBlockPointer())
     return true;
   if (!Ptr.getDeclID())
@@ -648,10 +658,24 @@ bool CheckMutable(InterpState &S, CodePtr OpPC, PtrView Ptr, AccessKinds AK) {
   if (!Ptr.isMutable())
     return true;
 
-  // In C++14 onwards, it is permitted to read a mutable member whose
-  // lifetime began within the evaluation.
-  if (S.getLangOpts().CPlusPlus14 && Ptr.getEvalID() == S.EvalID)
+  if (S.checkingConstantDestruction()) {
+    // Never allowed when checking for constant destruction.
+    // Find the reason this pointer is mutable.
+    PtrView MutablePtr = Ptr;
+    while (!MutablePtr.isRoot() && MutablePtr.getBase().isMutable())
+      MutablePtr = MutablePtr.getBase();
+
+    const FieldDecl *Field = MutablePtr.getField();
+    S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_access_mutable, 1)
+        << AK << Field;
+    S.Note(Field->getLocation(), diag::note_declared_at);
+    return false;
+  } else if (S.getLangOpts().CPlusPlus14 &&
+             S.lifetimeStartedInEvaluation(Ptr.block())) {
+    // In C++14 onwards, it is permitted to read a mutable member whose
+    // lifetime began within the evaluation.
     return true;
+  }
 
   const SourceInfo &Loc = S.Current->getSource(OpPC);
   const FieldDecl *Field = Ptr.getField();
@@ -985,21 +1009,23 @@ bool CheckStore(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
     return false;
   if (!CheckVolatile(S, OpPC, Ptr, AK_Assign))
     return false;
+  if (!CheckMutable(S, OpPC, Ptr, AK_Assign))
+    return false;
   if (isConstexprUnknown(Ptr))
     return false;
   return true;
 }
 
 static bool CheckInvoke(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
-                        bool IsCtorDtor = false) {
+                        bool IsCtor, bool IsDtor) {
   if (!Ptr.isDummy() && !isConstexprUnknown(Ptr)) {
     if (!CheckLive(S, OpPC, Ptr, AK_MemberCall))
       return false;
     if (!CheckRange(S, OpPC, Ptr, AK_MemberCall))
       return false;
-    if (!IsCtorDtor && !CheckLifetime(S, OpPC, Ptr, AK_MemberCall))
+    if (!(IsCtor || IsDtor) && !CheckLifetime(S, OpPC, Ptr, AK_MemberCall))
       return false;
-    if (!CheckMutable(S, OpPC, Ptr))
+    if (!IsDtor && !CheckMutable(S, OpPC, Ptr))
       return false;
   }
   return true;
@@ -1708,6 +1734,11 @@ bool CheckDestructor(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
   if (Ptr.getLifetime() == Lifetime::Ended)
     return CheckLifetime(S, OpPC, Ptr, AK_Destroy);
 
+  // We _can_ call the destructor on the global variable we're checking constant
+  // destruction for.
+  if (S.checkingConstantDestruction(Ptr))
+    return true;
+
   // Can't call a dtor on a global variable.
   if (Ptr.block()->isStatic()) {
     const SourceInfo &E = S.Current->getSource(OpPC);
@@ -1803,8 +1834,8 @@ bool CallVar(InterpState &S, CodePtr OpPC, const Function *Func,
     if (!(S.Current->getFunction() &&
           S.Current->getFunction()->isLambdaStaticInvoker() &&
           Func->isLambdaCallOperator())) {
-      if (!CheckInvoke(S, OpPC, ThisPtr,
-                       Func->isConstructor() || Func->isDestructor()))
+      if (!CheckInvoke(S, OpPC, ThisPtr, Func->isConstructor(),
+                       Func->isDestructor()))
         return false;
     }
 
@@ -1873,14 +1904,14 @@ bool Call(InterpState &S, CodePtr OpPC, const Function *Func,
         Func->isLambdaCallOperator()) {
       assert(ThisPtr.isZero());
     } else {
-      if (!CheckInvoke(S, OpPC, ThisPtr,
-                       Func->isConstructor() || Func->isDestructor()))
+      if (!CheckInvoke(S, OpPC, ThisPtr, Func->isConstructor(),
+                       Func->isDestructor()))
         return cleanup();
 
       if (Func->isCopyOrMoveOperator() || Func->isCopyOrMoveConstructor()) {
         const Pointer &RVOPtr =
             S.Stk.peek<Pointer>(ThisOffset - align(sizeof(Pointer)));
-        if (!CheckInvoke(S, OpPC, RVOPtr, /*IsCtorDtor=*/true))
+        if (!CheckInvoke(S, OpPC, RVOPtr, /*IsCtor=*/true, /*IsDtor=*/false))
           return cleanup();
       }
 
diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index ed640f7325f7e..aa2dffc2b982a 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -2052,6 +2052,14 @@ inline bool GetRefLocal(InterpState &S, CodePtr OpPC, uint32_t I) {
 inline bool GetRefGlobal(InterpState &S, CodePtr OpPC, uint32_t I) {
   Block *B = S.P.getGlobal(I);
 
+  // If we're currently evaluating this variable, use that in-flight value.
+  // It will otherwise be diagnosed as non-initialized reference and we will
+  // complain about a missing initializer.
+  if (S.EvaluatingDecl && B->getDescriptor()->asVarDecl() == S.EvaluatingDecl) {
+    S.Stk.push<Pointer>(B);
+    return true;
+  }
+
   if (isConstexprUnknown(B)) {
     S.Stk.push<Pointer>(B);
     return true;
diff --git a/clang/lib/AST/ByteCode/InterpFrame.cpp b/clang/lib/AST/ByteCode/InterpFrame.cpp
index 5eb8273af17f2..7366aca114ca0 100644
--- a/clang/lib/AST/ByteCode/InterpFrame.cpp
+++ b/clang/lib/AST/ByteCode/InterpFrame.cpp
@@ -287,6 +287,7 @@ SourceInfo InterpFrame::getSource(CodePtr PC) const {
   SourceInfo Result = S.getSource(Func, PC);
   if (Result.getLoc().isInvalid() && Caller)
     return Caller->getSource(RetPC);
+
   return Result;
 }
 
diff --git a/clang/lib/AST/ByteCode/InterpState.h b/clang/lib/AST/ByteCode/InterpState.h
index 92073cebcd490..4e4a053d6bbed 100644
--- a/clang/lib/AST/ByteCode/InterpState.h
+++ b/clang/lib/AST/ByteCode/InterpState.h
@@ -32,6 +32,13 @@ struct StdAllocatorCaller {
   explicit operator bool() { return Call; }
 };
 
+// FIXME: Create one for the "checking potential constant expression"
+// evaluation.
+enum class EvaluationKind : uint8_t {
+  None,
+  Dtor, /// We're checking for constant destruction of a global variable.
+};
+
 /// Interpreter context.
 class InterpState final : public State, public SourceMapper {
 public:
@@ -132,6 +139,31 @@ class InterpState final : public State, public SourceMapper {
     return false;
   }
 
+  bool lifetimeStartedInEvaluation(const Block *B) const {
+    if (EvalKind == EvaluationKind::None)
+      return B->getEvalID() == EvalID;
+
+    if (EvalKind == EvaluationKind::Dtor) {
+      assert(EvaluatingDecl);
+      if (B->getDescriptor()->asVarDecl() == EvaluatingDecl)
+        return EvaluatingDecl->getType().isConstQualified();
+    }
+    return false;
+  }
+
+  /// Return if we're checking if a global variable has a constant destructor.
+  bool checkingConstantDestruction() const {
+    return EvalKind == EvaluationKind::Dtor;
+  }
+  /// Return if we're checking if a global variable has a constant destructor
+  /// and the given pointer is pointing to the variable we're checking that for.
+  bool checkingConstantDestruction(const Pointer &Ptr) const {
+    return checkingConstantDestruction(Ptr.getDeclDesc()->asVarDecl());
+  }
+  bool checkingConstantDestruction(const VarDecl *VD) const {
+    return EvalKind == EvaluationKind::Dtor && VD == EvaluatingDecl;
+  }
+
 private:
   friend class EvaluationResult;
   friend class InterpStateCCOverride;
@@ -167,6 +199,8 @@ class InterpState final : public State, public SourceMapper {
   /// ID identifying this evaluation.
   const unsigned EvalID;
 
+  EvaluationKind EvalKind = EvaluationKind::None;
+
   /// Things needed to do speculative execution.
   SmallVectorImpl<PartialDiagnosticAt> *PrevDiags = nullptr;
   bool PrevDiagsEmitted = false;
diff --git a/clang/lib/AST/CommentLexer.cpp b/clang/lib/AST/CommentLexer.cpp
index a0903d0903dd8..a8ac08c90c630 100644
--- a/clang/lib/AST/CommentLexer.cpp
+++ b/clang/lib/AST/CommentLexer.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/CommentLexer.h"
+#include "clang/AST/Comment.h"
 #include "clang/AST/CommentCommandTraits.h"
 #include "clang/Basic/CharInfo.h"
 #include "clang/Basic/DiagnosticComment.h"
@@ -420,7 +421,10 @@ void Lexer::lexCommentText(Token &T) {
             << FullRange << CommandName << CorrectedName
             << FixItHint::CreateReplacement(CommandRange, CorrectedName);
         } else {
-          formTokenWithChars(T, TokenPtr, tok::unknown_command);
+          formTokenWithChars(T, TokenPtr,
+                             CommandKind == tok::backslash_command
+                                 ? tok::unknown_backslash_command
+                                 : tok::unknown_at_command);
           T.setUnknownCommandName(CommandName);
           Diag(T.getLocation(), diag::warn_unknown_comment_command_name)
               << SourceRange(T.getLocation(), T.getEndLocation());
diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp
index 68f18cfb5173e..87f8c870919d1 100644
--- a/clang/lib/AST/CommentParser.cpp
+++ b/clang/lib/AST/CommentParser.cpp
@@ -726,10 +726,12 @@ BlockContentComment *Parser::parseParagraphOrBlockCommand() {
     case tok::eof:
       break; // Block content or EOF ahead, finish this parapgaph.
 
-    case tok::unknown_command:
-      Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
-                                              Tok.getEndLocation(),
-                                              Tok.getUnknownCommandName()));
+    case tok::unknown_backslash_command:
+    case tok::unknown_at_command:
+      Content.push_back(S.actOnUnknownCommand(
+          Tok.getLocation(), Tok.getEndLocation(), Tok.getUnknownCommandName(),
+          Tok.getKind() == tok::unknown_backslash_command ? CMK_Backslash
+                                                          : CMK_At));
       consumeToken();
       continue;
 
@@ -751,9 +753,9 @@ BlockContentComment *Parser::parseParagraphOrBlockCommand() {
         continue;
       }
       if (Info->IsUnknownCommand) {
-        Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
-                                                Tok.getEndLocation(),
-                                                Info->getID()));
+        Content.push_back(S.actOnUnknownCommand(
+            Tok.getLocation(), Tok.getEndLocation(), Info->getID(),
+            Tok.getKind() == tok::backslash_command ? CMK_Backslash : CMK_At));
         consumeToken();
         continue;
       }
@@ -892,7 +894,8 @@ VerbatimLineComment *Parser::parseVerbatimLine() {
 BlockContentComment *Parser::parseBlockContent() {
   switch (Tok.getKind()) {
   case tok::text:
-  case tok::unknown_command:
+  case tok::unknown_backslash_command:
+  case tok::unknown_at_command:
   case tok::backslash_command:
   case tok::at_command:
   case tok::html_start_tag:
diff --git a/clang/lib/AST/CommentSema.cpp b/clang/lib/AST/CommentSema.cpp
index e74c7cb5ce605..bc7884f1ffd52 100644
--- a/clang/lib/AST/CommentSema.cpp
+++ b/clang/lib/AST/CommentSema.cpp
@@ -376,19 +376,21 @@ Sema::actOnInlineCommand(SourceLocation CommandLocBegin,
       getInlineCommandRenderKind(CommandName), CommandMarker, Args);
 }
 
-InlineContentComment *Sema::actOnUnknownCommand(SourceLocation LocBegin,
-                                                SourceLocation LocEnd,
-                                                StringRef CommandName) {
+InlineContentComment *
+Sema::actOnUnknownCommand(SourceLocation LocBegin, SourceLocation LocEnd,
+                          StringRef CommandName,
+                          CommandMarkerKind CommandMarker) {
   unsigned CommandID = Traits.registerUnknownCommand(CommandName)->getID();
-  return actOnUnknownCommand(LocBegin, LocEnd, CommandID);
+  return actOnUnknownCommand(LocBegin, LocEnd, CommandID, CommandMarker);
 }
 
-InlineContentComment *Sema::actOnUnknownCommand(SourceLocation LocBegin,
-                                                SourceLocation LocEnd,
-                                                unsigned CommandID) {
+InlineContentComment *
+Sema::actOnUnknownCommand(SourceLocation LocBegin, SourceLocation LocEnd,
+                          unsigned CommandID, CommandMarkerKind CommandMarker) {
   ArrayRef<InlineCommandComment::Argument> Args;
-  return new (Allocator) InlineCommandComment(
-      LocBegin, LocEnd, CommandID, InlineCommandRenderKind::Normal, Args);
+  return new (Allocator) InlineCommandComment(LocBegin, LocEnd, CommandID,
+                                              InlineCommandRenderKind::Normal,
+                                              CommandMarker, Args);
 }
 
 TextComment *Sema::actOnText(SourceLocation LocBegin,
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 28ac44edd800c..1d359339b9104 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -21748,8 +21748,6 @@ bool VarDecl::evaluateDestruction(
     EvalInfo Info(Ctx, EStatus,
                   IsConstantDestruction ? EvaluationMode::ConstantExpression
                                         : EvaluationMode::ConstantFold);
-    Info.setEvaluatingDecl(this, DestroyedValue,
-                           EvalInfo::EvaluatingDeclKind::Dtor);
     Info.InConstantContext = IsConstantDestruction;
     if (!Ctx.getInterpContext().evaluateDestruction(Info, this,
                                                     std::move(DestroyedValue)))
diff --git a/clang/lib/Analysis/FlowSensitive/WatchedLiteralsSolver.cpp b/clang/lib/Analysis/FlowSensitive/WatchedLiteralsSolver.cpp
index a39f0e0b29ad1..98109faff222a 100644
--- a/clang/lib/Analysis/FlowSensitive/WatchedLiteralsSolver.cpp
+++ b/clang/lib/Analysis/FlowSensitive/WatchedLiteralsSolver.cpp
@@ -162,9 +162,11 @@ class WatchedLiteralsSolverImpl {
       // FIXME: Consider replacing these with test cases that fail if the any
       // of the invariants is broken. That might not be easy due to the
       // transformations performed by `buildCNF`.
+#ifdef EXPENSIVE_CHECKS
       assert(activeVarsAreUnassigned());
       assert(activeVarsFormWatchedLiterals());
       assert(unassignedVarsFormingWatchedLiteralsAreActive());
+#endif
 
       const Variable ActiveVar = ActiveVars[I];
 
diff --git a/clang/lib/Analysis/LifetimeSafety/Checker.cpp b/clang/lib/Analysis/LifetimeSafety/Checker.cpp
index c258a1dc3596c..746ebbfb15c39 100644
--- a/clang/lib/Analysis/LifetimeSafety/Checker.cpp
+++ b/clang/lib/Analysis/LifetimeSafety/Checker.cpp
@@ -68,6 +68,7 @@ class LifetimeChecker {
   LifetimeSafetySemaHelper *SemaHelper;
   ASTContext &AST;
   const Decl *FD;
+  const LifetimeSafetyOpts &LSOpts;
 
   static SourceLocation
   GetFactLoc(llvm::PointerUnion<const UseFact *, const OriginEscapesFact *> F) {
@@ -87,10 +88,11 @@ class LifetimeChecker {
                   const MovedLoansAnalysis &MovedLoans,
                   const LiveOriginsAnalysis &LiveOrigins, FactManager &FM,
                   AnalysisDeclContext &ADC,
-                  LifetimeSafetySemaHelper *SemaHelper)
+                  LifetimeSafetySemaHelper *SemaHelper,
+                  const LifetimeSafetyOpts &LSOpts)
       : LoanPropagation(LoanPropagation), MovedLoans(MovedLoans),
         LiveOrigins(LiveOrigins), FactMgr(FM), SemaHelper(SemaHelper),
-        AST(ADC.getASTContext()), FD(ADC.getDecl()) {
+        AST(ADC.getASTContext()), FD(ADC.getDecl()), LSOpts(LSOpts) {
     for (const CFGBlock *B : *ADC.getAnalysis<PostOrderCFGView>())
       for (const Fact *F : FactMgr.getFacts(B))
         if (const auto *EF = F->getAs<ExpireFact>())
@@ -398,6 +400,9 @@ class LifetimeChecker {
   void suggestAnnotations() {
     if (!SemaHelper)
       return;
+    if (!LSOpts.SuggestAnnotations)
+      return;
+    llvm::TimeTraceScope TimeTrace("SuggestAnnotations");
     for (auto [Target, EscapeTarget] : AnnotationWarningsMap) {
       if (const auto *PVD = Target.dyn_cast<const ParmVarDecl *>())
         suggestWithScopeForParmVar(PVD, EscapeTarget);
@@ -535,9 +540,10 @@ void runLifetimeChecker(const LoanPropagationAnalysis &LP,
                         const MovedLoansAnalysis &MovedLoans,
                         const LiveOriginsAnalysis &LO, FactManager &FactMgr,
                         AnalysisDeclContext &ADC,
-                        LifetimeSafetySemaHelper *SemaHelper) {
+                        LifetimeSafetySemaHelper *SemaHelper,
+                        const LifetimeSafetyOpts &LSOpts) {
   llvm::TimeTraceScope TimeProfile("LifetimeChecker");
-  LifetimeChecker Checker(LP, MovedLoans, LO, FactMgr, ADC, SemaHelper);
+  LifetimeChecker Checker(LP, MovedLoans, LO, FactMgr, ADC, SemaHelper, LSOpts);
 }
 
 } // namespace clang::lifetimes::internal
diff --git a/clang/lib/Analysis/LifetimeSafety/LifetimeSafety.cpp b/clang/lib/Analysis/LifetimeSafety/LifetimeSafety.cpp
index 714f979fa5ee7..680095a54177c 100644
--- a/clang/lib/Analysis/LifetimeSafety/LifetimeSafety.cpp
+++ b/clang/lib/Analysis/LifetimeSafety/LifetimeSafety.cpp
@@ -96,7 +96,7 @@ void LifetimeSafetyAnalysis::run() {
       Factory.MovedLoansMapFactory);
 
   runLifetimeChecker(*LoanPropagation, *MovedLoans, *LiveOrigins, *FactMgr, AC,
-                     SemaHelper);
+                     SemaHelper, LSOpts);
 
   DEBUG_WITH_TYPE("PrintCFG", Cfg.dump(AC.getASTContext().getLangOpts(),
                                        /*ShowColors=*/true));
@@ -122,11 +122,8 @@ void collectLifetimeStats(AnalysisDeclContext &AC, OriginManager &OM,
 
 void runLifetimeSafetyAnalysis(AnalysisDeclContext &AC,
                                LifetimeSafetySemaHelper *SemaHelper,
+                               const LifetimeSafetyOpts &LSOpts,
                                LifetimeSafetyStats &Stats, bool CollectStats) {
-  LifetimeSafetyOpts LSOpts;
-  LSOpts.MaxCFGBlocks =
-      AC.getASTContext().getLangOpts().LifetimeSafetyMaxCFGBlocks;
-
   internal::LifetimeSafetyAnalysis Analysis(AC, SemaHelper, LSOpts);
   Analysis.run();
   if (CollectStats)
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprCXX.cpp b/clang/lib/CIR/CodeGen/CIRGenExprCXX.cpp
index 1a565b077a2eb..df60005d77259 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprCXX.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprCXX.cpp
@@ -1068,10 +1068,10 @@ void CIRGenFunction::emitNewArrayInitializer(
       remainingSize = builder.createSub(loc, remainingSize, initSizeOp);
     }
 
-    // Create the memset.
-    mlir::Value castOp =
-        builder.createPtrBitcast(curPtr.getPointer(), cgm.voidTy);
-    builder.createMemSet(loc, castOp, builder.getConstInt(loc, cgm.uInt8Ty, 0),
+    // Create the memset.  Use the Address overload so the destination
+    // alignment from curPtr is carried onto the memset.
+    Address voidPtr = curPtr.withElementType(builder, cgm.voidTy);
+    builder.createMemSet(loc, voidPtr, builder.getConstInt(loc, cgm.uInt8Ty, 0),
                          remainingSize);
     return true;
   };
@@ -1228,12 +1228,11 @@ void CIRGenFunction::emitNewArrayInitializer(
     if (ctor->isTrivial()) {
       // If new expression did not specify value-initialization, then there
       // is no initialization.
-      if (!cce->requiresZeroInitialization())
+      if (!cce->requiresZeroInitialization() || ctor->getParent()->isEmpty())
         return;
 
-      cgm.errorNYI(cce->getSourceRange(),
-                   "emitNewArrayInitializer: trivial ctor zero-init");
-      return;
+      if (tryMemsetInitialization())
+        return;
     }
 
     // Store the new Cleanup position for irregular Cleanups.
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index b6a4a277fab92..0ce6005ccac88 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -2566,10 +2566,6 @@ class CIRGenFunction : public CIRGenTypeCache {
   void emitOMPDeclareMapper(const OMPDeclareMapperDecl &d);
   void emitOMPRequiresDecl(const OMPRequiresDecl &d);
 
-private:
-  template <typename Op>
-  void emitOpenMPClauses(Op &op, ArrayRef<const OMPClause *> clauses);
-
   //===--------------------------------------------------------------------===//
   //                         OpenACC Emission
   //===--------------------------------------------------------------------===//
diff --git a/clang/lib/CIR/CodeGen/CIRGenOpenMPClause.cpp b/clang/lib/CIR/CodeGen/CIRGenOpenMPClause.cpp
index a0f0ea9299c8d..16ac4440660b5 100644
--- a/clang/lib/CIR/CodeGen/CIRGenOpenMPClause.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenOpenMPClause.cpp
@@ -6,90 +6,140 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// Emit OpenMP clause nodes as CIR code.
+// OpenMP clause emitter implementation.
 //
 //===----------------------------------------------------------------------===//
 
+#include "CIRGenOpenMPClause.h"
 #include "CIRGenFunction.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
+#include "clang/Basic/OpenMPKinds.h"
 
 using namespace clang;
 using namespace clang::CIRGen;
 
-namespace {
-template <typename OpTy>
-class OpenMPClauseCIREmitter final
-    : public ConstOMPClauseVisitor<OpenMPClauseCIREmitter<OpTy>> {
-  OpTy &operation;
-  CIRGen::CIRGenFunction &cgf;
-  CIRGen::CIRGenBuilderTy &builder;
-
-public:
-  OpenMPClauseCIREmitter(OpTy &operation, CIRGen::CIRGenFunction &cgf,
-                         CIRGen::CIRGenBuilderTy &builder)
-      : operation(operation), cgf(cgf), builder(builder) {}
-
-  void VisitOMPClause(const OMPClause *clause) {
-    cgf.cgm.errorNYI(clause->getBeginLoc(), "OpenMPClause ",
-                     llvm::omp::getOpenMPClauseName(clause->getClauseKind()));
+static mlir::omp::ClauseMapFlags
+mapClauseKindToFlags(OpenMPMapClauseKind kind) {
+  switch (kind) {
+  case OMPC_MAP_to:
+    return mlir::omp::ClauseMapFlags::to;
+  case OMPC_MAP_from:
+    return mlir::omp::ClauseMapFlags::from;
+  case OMPC_MAP_tofrom:
+    return mlir::omp::ClauseMapFlags::to | mlir::omp::ClauseMapFlags::from;
+  case OMPC_MAP_alloc:
+  case OMPC_MAP_release:
+    return mlir::omp::ClauseMapFlags::storage;
+  case OMPC_MAP_delete:
+    return mlir::omp::ClauseMapFlags::del;
+  default:
+    return mlir::omp::ClauseMapFlags::none;
   }
+}
 
-  void VisitOMPProcBindClause(const OMPProcBindClause *clause) {
-    if constexpr (std::is_same_v<OpTy, mlir::omp::ParallelOp>) {
-      mlir::omp::ClauseProcBindKind kind;
-      switch (clause->getProcBindKind()) {
-      case llvm::omp::ProcBindKind::OMP_PROC_BIND_master:
-        kind = mlir::omp::ClauseProcBindKind::Master;
-        break;
-      case llvm::omp::ProcBindKind::OMP_PROC_BIND_close:
-        kind = mlir::omp::ClauseProcBindKind::Close;
-        break;
-      case llvm::omp::ProcBindKind::OMP_PROC_BIND_spread:
-        kind = mlir::omp::ClauseProcBindKind::Spread;
-        break;
-      case llvm::omp::ProcBindKind::OMP_PROC_BIND_primary:
-        kind = mlir::omp::ClauseProcBindKind::Primary;
-        break;
-      case llvm::omp::ProcBindKind::OMP_PROC_BIND_default:
-        // 'default' in the classic-codegen does no runtime call/doesn't
-        // really do anything. So this is a no-op, and thus shouldn't change
-        // the IR.
-        return;
-      case llvm::omp::ProcBindKind::OMP_PROC_BIND_unknown:
-        llvm_unreachable("unknown proc-bind kind");
-      }
-      operation.setProcBindKind(kind);
-    } else {
-      cgf.cgm.errorNYI(
-          clause->getBeginLoc(),
-          "OMPProcBindClause unimplemented on this directive kind");
-    }
-  }
+static mlir::Value emitMapInfoForVar(CIRGenFunction &cgf,
+                                     mlir::OpBuilder &builder,
+                                     mlir::Location loc, const VarDecl *vd,
+                                     mlir::omp::ClauseMapFlags mapFlags) {
+  Address addr = cgf.getAddrOfLocalVar(vd);
+  mlir::Value varPtr = addr.getPointer();
+  auto varPtrType = mlir::cast<cir::PointerType>(varPtr.getType());
+  mlir::Type elementType = varPtrType.getPointee();
 
-  void emitClauses(ArrayRef<const OMPClause *> clauses) {
-    for (const auto *c : clauses)
-      this->Visit(c);
+  // Cast to generic pointer if needed.
+  if (varPtrType.getAddrSpace()) {
+    auto genericPtrType =
+        cir::PointerType::get(builder.getContext(), elementType);
+    varPtr = cir::CastOp::create(builder, loc, genericPtrType,
+                                 cir::CastKind::address_space, varPtr);
+    varPtrType = genericPtrType;
   }
-};
-template <typename OpTy>
-auto makeClauseEmitter(OpTy &op, CIRGen::CIRGenFunction &cgf,
-                       CIRGen::CIRGenBuilderTy &builder) {
-  return OpenMPClauseCIREmitter<OpTy>(op, cgf, builder);
+
+  return mlir::omp::MapInfoOp::create(
+      builder, loc,
+      /*omp_ptr=*/varPtrType,
+      /*var_ptr=*/varPtr,
+      /*var_ptr_type=*/mlir::TypeAttr::get(elementType),
+      /*map_type=*/builder.getAttr<mlir::omp::ClauseMapFlagsAttr>(mapFlags),
+      /*map_capture_type=*/
+      builder.getAttr<mlir::omp::VariableCaptureKindAttr>(
+          mlir::omp::VariableCaptureKind::ByRef),
+      /*var_ptr_ptr=*/mlir::Value{},
+      /*var_ptr_ptr_type=*/mlir::TypeAttr{},
+      /*members=*/mlir::ValueRange{},
+      /*members_index=*/mlir::ArrayAttr{},
+      /*bounds=*/mlir::ValueRange{},
+      /*mapper_id=*/mlir::FlatSymbolRefAttr{},
+      /*name=*/builder.getStringAttr(vd->getName()),
+      /*partial_map=*/builder.getBoolAttr(false));
 }
-} // namespace
-
-template <typename Op>
-void CIRGenFunction::emitOpenMPClauses(Op &op,
-                                       ArrayRef<const OMPClause *> clauses) {
-  mlir::OpBuilder::InsertionGuard guardCase(builder);
-  builder.setInsertionPoint(op);
-  makeClauseEmitter(op, *this, builder).emitClauses(clauses);
+
+bool OpenMPClauseEmitter::emitProcBind(
+    mlir::omp::ProcBindClauseOps &result) const {
+  for (const OMPClause *clause : clauses) {
+    const auto *pbc = dyn_cast<OMPProcBindClause>(clause);
+    if (!pbc)
+      continue;
+
+    llvm::omp::ProcBindKind kind = pbc->getProcBindKind();
+    assert(kind != llvm::omp::ProcBindKind::OMP_PROC_BIND_unknown &&
+           "unknown proc-bind kind");
+    // The 'default' kind has no dialect counterpart; leave the attribute unset.
+    if (kind != llvm::omp::ProcBindKind::OMP_PROC_BIND_default)
+      result.procBindKind = mlir::omp::ClauseProcBindKindAttr::get(
+          builder.getContext(), mlir::omp::convertProcBindKind(kind));
+    return true;
+  }
+  return false;
 }
 
-// We're defining the template for this in a .cpp file, so we have to explicitly
-// specialize the templates.
-#define EXPL_SPEC(N)                                                           \
-  template void CIRGenFunction::emitOpenMPClauses<N>(                          \
-      N &, ArrayRef<const OMPClause *>);
-EXPL_SPEC(mlir::omp::ParallelOp)
-#undef EXPL_SPEC
+bool OpenMPClauseEmitter::emitMap(
+    mlir::omp::MapClauseOps &result,
+    llvm::SmallVectorImpl<const VarDecl *> *mapSyms) const {
+  bool found = false;
+  for (const OMPClause *clause : clauses) {
+    const auto *mc = dyn_cast<OMPMapClause>(clause);
+    if (!mc)
+      continue;
+
+    found = true;
+
+    for (OpenMPMapModifierKind mod : mc->getMapTypeModifiers()) {
+      if (mod != OMPC_MAP_MODIFIER_unknown)
+        cgm.errorNYI(mc->getBeginLoc(),
+                     std::string("OpenMP map modifier '") +
+                         getOpenMPSimpleClauseTypeName(
+                             llvm::omp::Clause::OMPC_map, mod) +
+                         "'");
+    }
+
+    if (mc->isImplicit()) {
+      cgm.errorNYI(mc->getBeginLoc(), "OpenMP implicit map clause");
+      continue;
+    }
+
+    mlir::omp::ClauseMapFlags mapFlags = mapClauseKindToFlags(mc->getMapType());
+
+    for (const Expr *varExpr : mc->varlist()) {
+      const auto *refExpr = dyn_cast<DeclRefExpr>(varExpr->IgnoreImplicit());
+      if (!refExpr) {
+        cgm.errorNYI(varExpr->getExprLoc(),
+                     "OpenMP map clause with non-DeclRefExpr variable");
+        continue;
+      }
+
+      const auto *vd = dyn_cast<VarDecl>(refExpr->getDecl());
+      if (!vd) {
+        cgm.errorNYI(varExpr->getExprLoc(),
+                     "OpenMP map clause with non-VarDecl variable");
+        continue;
+      }
+
+      result.mapVars.push_back(
+          emitMapInfoForVar(cgf, builder, loc, vd, mapFlags));
+      if (mapSyms)
+        mapSyms->push_back(vd);
+    }
+  }
+  return found;
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenOpenMPClause.h b/clang/lib/CIR/CodeGen/CIRGenOpenMPClause.h
new file mode 100644
index 0000000000000..54c7366b1d769
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/CIRGenOpenMPClause.h
@@ -0,0 +1,88 @@
+//===--- CIRGenOpenMPClause.h - OpenMP clause emitter -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_CIR_CODEGEN_CIRGENOPENMPCLAUSE_H
+#define LLVM_CLANG_LIB_CIR_CODEGEN_CIRGENOPENMPCLAUSE_H
+
+#include "CIRGenBuilder.h"
+#include "CIRGenModule.h"
+#include "mlir/Dialect/OpenMP/OpenMPClauseOperands.h"
+#include "clang/AST/OpenMPClause.h"
+#include "clang/AST/StmtOpenMP.h"
+#include "llvm/Frontend/OpenMP/OMPConstants.h"
+
+#include <type_traits>
+
+namespace clang::CIRGen {
+
+class CIRGenFunction;
+
+/// A type-only list of OpenMP clause AST node types.
+template <typename... Clauses> struct OpenMPNYIClauseList {};
+
+/// Emits OpenMP clauses for a directive, writing results into the
+/// auto-generated ClauseOps from the OMP dialect.
+class OpenMPClauseEmitter {
+  CIRGenFunction &cgf;
+  CIRGenModule &cgm;
+  CIRGenBuilderTy &builder;
+  mlir::Location loc;
+  llvm::ArrayRef<const OMPClause *> clauses;
+
+public:
+  OpenMPClauseEmitter(CIRGenFunction &cgf, CIRGenModule &cgm,
+                      CIRGenBuilderTy &builder, mlir::Location loc,
+                      llvm::ArrayRef<const OMPClause *> clauses)
+      : cgf(cgf), cgm(cgm), builder(builder), loc(loc), clauses(clauses) {}
+
+  bool emitProcBind(mlir::omp::ProcBindClauseOps &result) const;
+
+  /// Emit map clauses. The optional \p mapSyms parameter collects the
+  /// VarDecls corresponding to each map operand.
+  bool emitMap(mlir::omp::MapClauseOps &result,
+               llvm::SmallVectorImpl<const VarDecl *> *mapSyms = nullptr) const;
+
+  /// Verify the clauses of a directive to make sure all legal cases are either
+  /// implemented or give a NYI error. The \p SupportedClauses and \p
+  /// NYIClauses type lists must be disjoint and cover all clauses eligible for
+  /// the directive being processed.
+  template <typename... SupportedClauses, typename... NYIClauses>
+  void emitNYI(OpenMPNYIClauseList<NYIClauses...> nyi,
+               llvm::omp::Directive directive) const;
+
+private:
+  /// True if T is the same type as any of Ts.
+  template <typename T, typename... Ts>
+  static constexpr bool isAnyOf = (std::is_same_v<T, Ts> || ...);
+};
+
+template <typename... SupportedClauses, typename... NYIClauses>
+void OpenMPClauseEmitter::emitNYI(OpenMPNYIClauseList<NYIClauses...>,
+                                  llvm::omp::Directive directive) const {
+  static_assert(
+      (!isAnyOf<NYIClauses, SupportedClauses...> && ...),
+      "the supported and not-yet-implemented clause lists must be disjoint");
+
+  for (const OMPClause *c : clauses) {
+    if (isa<NYIClauses...>(c)) {
+      std::string msg =
+          (llvm::Twine("OpenMP ") +
+           llvm::omp::getOpenMPDirectiveName(directive).upper() + " '" +
+           llvm::omp::getOpenMPClauseName(c->getClauseKind()) + "' clause")
+              .str();
+      cgm.errorNYI(c->getBeginLoc(), msg);
+    } else if (!isa<SupportedClauses...>(c)) {
+      // Unknown/illegal clause encountered.
+      llvm_unreachable("unexpected OpenMP clause");
+    }
+  }
+}
+
+} // namespace clang::CIRGen
+
+#endif // LLVM_CLANG_LIB_CIR_CODEGEN_CIRGENOPENMPCLAUSE_H
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
index 47c94cb4ec535..a2c999f584399 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
@@ -421,7 +421,6 @@ mlir::LogicalResult CIRGenFunction::emitStmt(const Stmt *s,
   case Stmt::CaseStmtClass:
   case Stmt::SEHLeaveStmtClass:
   case Stmt::SYCLKernelCallStmtClass:
-  case Stmt::CapturedStmtClass:
   case Stmt::ObjCAtTryStmtClass:
   case Stmt::ObjCAtThrowStmtClass:
   case Stmt::ObjCAtSynchronizedStmtClass:
@@ -434,6 +433,8 @@ mlir::LogicalResult CIRGenFunction::emitStmt(const Stmt *s,
     cgm.errorNYI(s->getSourceRange(),
                  std::string("emitStmt: ") + s->getStmtClassName());
     return mlir::failure();
+  case Stmt::CapturedStmtClass:
+    llvm_unreachable("CapturedStmt must be handled by the parent directive");
   }
 
   llvm_unreachable("Unexpected statement class");
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp b/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
index eb4934644b519..17a1fb8090f5c 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
@@ -12,7 +12,9 @@
 
 #include "CIRGenBuilder.h"
 #include "CIRGenFunction.h"
+#include "CIRGenOpenMPClause.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
+#include "clang/AST/OpenMPClause.h"
 #include "clang/AST/StmtOpenMP.h"
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
 using namespace clang;
@@ -31,14 +33,20 @@ CIRGenFunction::emitOMPErrorDirective(const OMPErrorDirective &s) {
 mlir::LogicalResult
 CIRGenFunction::emitOMPParallelDirective(const OMPParallelDirective &s) {
   mlir::LogicalResult res = mlir::success();
-  llvm::SmallVector<mlir::Type> retTy;
-  llvm::SmallVector<mlir::Value> operands;
   mlir::Location begin = getLoc(s.getBeginLoc());
   mlir::Location end = getLoc(s.getEndLoc());
 
-  auto parallelOp =
-      mlir::omp::ParallelOp::create(builder, begin, retTy, operands);
-  emitOpenMPClauses(parallelOp, s.clauses());
+  mlir::omp::ParallelOperands clauseOps;
+  OpenMPClauseEmitter ce(*this, getCIRGenModule(), builder, begin, s.clauses());
+  ce.emitProcBind(clauseOps);
+  ce.emitNYI</*supported=*/OMPProcBindClause>(
+      /*nyi=*/OpenMPNYIClauseList<
+          OMPAllocateClause, OMPCopyinClause, OMPDefaultClause,
+          OMPFirstprivateClause, OMPIfClause, OMPNumThreadsClause,
+          OMPPrivateClause, OMPReductionClause, OMPSharedClause>{},
+      llvm::omp::Directive::OMPD_parallel);
+
+  auto parallelOp = mlir::omp::ParallelOp::create(builder, begin, clauseOps);
 
   {
     mlir::Block &block = parallelOp.getRegion().emplaceBlock();
@@ -207,10 +215,108 @@ CIRGenFunction::emitOMPAtomicDirective(const OMPAtomicDirective &s) {
   getCIRGenModule().errorNYI(s.getSourceRange(), "OpenMP OMPAtomicDirective");
   return mlir::failure();
 }
+
+/// Check for unsupported implicit captures in a target region.
+static void
+emitOMPTargetImplicitCaptures(CIRGenFunction &cgf, const OMPTargetDirective &s,
+                              llvm::ArrayRef<const VarDecl *> mapSyms) {
+  const CapturedStmt *cs = s.getCapturedStmt(llvm::omp::OMPD_target);
+  for (const auto &capture : cs->captures()) {
+    if (capture.capturesThis()) {
+      cgf.getCIRGenModule().errorNYI(s.getBeginLoc(),
+                                     "OpenMP target capture of 'this' pointer");
+      continue;
+    }
+    if (capture.capturesVariableByCopy()) {
+      cgf.getCIRGenModule().errorNYI(s.getBeginLoc(),
+                                     "OpenMP target capture by copy");
+      continue;
+    }
+    if (capture.capturesVariableArrayType()) {
+      cgf.getCIRGenModule().errorNYI(
+          s.getBeginLoc(),
+          "OpenMP target capture of variable-length array type");
+      continue;
+    }
+    if (capture.capturesVariable()) {
+      const VarDecl *vd = capture.getCapturedVar();
+      if (llvm::is_contained(mapSyms, vd))
+        continue;
+
+      cgf.getCIRGenModule().errorNYI(s.getBeginLoc(),
+                                     "OpenMP target implicit by-ref capture");
+    }
+  }
+}
+
+/// Emit the body of an omp.target region, remapping mapped variables to the
+/// block arguments of the target op's region.
+static mlir::LogicalResult
+emitOMPTargetBody(CIRGenFunction &cgf, const OMPTargetDirective &s,
+                  mlir::omp::TargetOp targetOp,
+                  llvm::ArrayRef<mlir::Value> mapVars,
+                  llvm::ArrayRef<const VarDecl *> mappedVarDecls,
+                  mlir::Location begin, mlir::Location end) {
+  mlir::Block &block = targetOp.getRegion().emplaceBlock();
+
+  for (mlir::Value mapVar : mapVars)
+    block.addArgument(mapVar.getType(), begin);
+
+  mlir::OpBuilder::InsertionGuard guard(cgf.getBuilder());
+  cgf.getBuilder().setInsertionPointToEnd(&block);
+
+  CIRGenFunction::LexicalScope ls{cgf, begin,
+                                  cgf.getBuilder().getInsertionBlock()};
+
+  llvm::SmallVector<std::pair<const VarDecl *, Address>> savedAddrs;
+  for (auto [idx, vd] : llvm::enumerate(mappedVarDecls)) {
+    Address origAddr = cgf.getAddrOfLocalVar(vd);
+    savedAddrs.push_back({vd, origAddr});
+    mlir::Value blockArg = block.getArgument(idx);
+    cgf.replaceAddrOfLocalVar(vd, Address(blockArg, origAddr.getAlignment()));
+  }
+
+  const CapturedStmt *cs = s.getCapturedStmt(llvm::omp::OMPD_target);
+  mlir::LogicalResult res =
+      cgf.emitStmt(cs->getCapturedStmt(), /*useCurrentScope=*/true);
+
+  mlir::omp::TerminatorOp::create(cgf.getBuilder(), end);
+
+  for (auto &[vd, addr] : savedAddrs)
+    cgf.replaceAddrOfLocalVar(vd, addr);
+
+  return res;
+}
+
 mlir::LogicalResult
 CIRGenFunction::emitOMPTargetDirective(const OMPTargetDirective &s) {
-  getCIRGenModule().errorNYI(s.getSourceRange(), "OpenMP OMPTargetDirective");
-  return mlir::failure();
+  mlir::Location begin = getLoc(s.getBeginLoc());
+  mlir::Location end = getLoc(s.getEndLoc());
+
+  mlir::omp::TargetExtOperands clauseOps;
+  llvm::SmallVector<const VarDecl *> mapSyms;
+
+  OpenMPClauseEmitter ce(*this, getCIRGenModule(), builder, begin, s.clauses());
+  ce.emitMap(clauseOps, &mapSyms);
+  ce.emitNYI</*supported=*/OMPMapClause>(
+      /*nyi=*/OpenMPNYIClauseList<
+          OMPAllocateClause, OMPDefaultClause, OMPDefaultmapClause,
+          OMPDependClause, OMPDeviceClause, OMPFirstprivateClause,
+          OMPHasDeviceAddrClause, OMPIfClause, OMPInReductionClause,
+          OMPIsDevicePtrClause, OMPNowaitClause, OMPPrivateClause,
+          OMPThreadLimitClause, OMPUsesAllocatorsClause, OMPXBareClause>{},
+      llvm::omp::Directive::OMPD_target);
+
+  emitOMPTargetImplicitCaptures(*this, s, mapSyms);
+
+  // Use generic for now.
+  clauseOps.kernelType = mlir::omp::TargetExecModeAttr::get(
+      &getMLIRContext(), mlir::omp::TargetExecMode::generic);
+
+  auto targetOp = mlir::omp::TargetOp::create(builder, begin, clauseOps);
+
+  return emitOMPTargetBody(*this, s, targetOp, clauseOps.mapVars, mapSyms,
+                           begin, end);
 }
 mlir::LogicalResult
 CIRGenFunction::emitOMPTeamsDirective(const OMPTeamsDirective &s) {
diff --git a/clang/lib/Headers/__clang_hip_libdevice_declares.h b/clang/lib/Headers/__clang_hip_libdevice_declares.h
index 565c68334023a..ad40e76f5d861 100644
--- a/clang/lib/Headers/__clang_hip_libdevice_declares.h
+++ b/clang/lib/Headers/__clang_hip_libdevice_declares.h
@@ -29,9 +29,6 @@ __device__ __attribute__((const)) float __ocml_atan2_f32(float, float);
 __device__ __attribute__((const)) float __ocml_atan_f32(float);
 __device__ __attribute__((pure)) float __ocml_atanh_f32(float);
 __device__ __attribute__((pure)) float __ocml_cbrt_f32(float);
-__device__ __attribute__((const)) float __ocml_ceil_f32(float);
-__device__ __attribute__((const)) __device__ float __ocml_copysign_f32(float,
-                                                                       float);
 __device__ float __ocml_cos_f32(float);
 __device__ float __ocml_native_cos_f32(float);
 __device__ __attribute__((pure)) __device__ float __ocml_cosh_f32(float);
@@ -43,40 +40,17 @@ __device__ __attribute__((pure)) float __ocml_erfcinv_f32(float);
 __device__ __attribute__((pure)) float __ocml_erfcx_f32(float);
 __device__ __attribute__((pure)) float __ocml_erf_f32(float);
 __device__ __attribute__((pure)) float __ocml_erfinv_f32(float);
-__device__ __attribute__((pure)) float __ocml_exp10_f32(float);
-__device__ __attribute__((pure)) float __ocml_native_exp10_f32(float);
-__device__ __attribute__((pure)) float __ocml_exp2_f32(float);
-__device__ __attribute__((pure)) float __ocml_exp_f32(float);
-__device__ __attribute__((pure)) float __ocml_native_exp_f32(float);
 __device__ __attribute__((pure)) float __ocml_expm1_f32(float);
-__device__ __attribute__((const)) float __ocml_fabs_f32(float);
 __device__ __attribute__((const)) float __ocml_fdim_f32(float, float);
-__device__ __attribute__((const)) float __ocml_floor_f32(float);
-__device__ __attribute__((const)) float __ocml_fma_f32(float, float, float);
-__device__ __attribute__((const)) float __ocml_fmax_f32(float, float);
-__device__ __attribute__((const)) float __ocml_fmin_f32(float, float);
 __device__ __attribute__((const)) __device__ float __ocml_fmod_f32(float,
                                                                    float);
-__device__ float __ocml_frexp_f32(float, __PRIVATE_AS int *);
 __device__ __attribute__((const)) float __ocml_hypot_f32(float, float);
 __device__ __attribute__((const)) int __ocml_ilogb_f32(float);
-__device__ __attribute__((const)) int __ocml_isfinite_f32(float);
-__device__ __attribute__((const)) int __ocml_isinf_f32(float);
-__device__ __attribute__((const)) int __ocml_isnan_f32(float);
 __device__ float __ocml_j0_f32(float);
 __device__ float __ocml_j1_f32(float);
-__device__ __attribute__((const)) float __ocml_ldexp_f32(float, int);
 __device__ float __ocml_lgamma_f32(float);
-__device__ __attribute__((pure)) float __ocml_log10_f32(float);
-__device__ __attribute__((pure)) float __ocml_native_log10_f32(float);
 __device__ __attribute__((pure)) float __ocml_log1p_f32(float);
-__device__ __attribute__((pure)) float __ocml_log2_f32(float);
-__device__ __attribute__((pure)) float __ocml_native_log2_f32(float);
-__device__ __attribute__((const)) float __ocml_logb_f32(float);
-__device__ __attribute__((pure)) float __ocml_log_f32(float);
-__device__ __attribute__((pure)) float __ocml_native_log_f32(float);
 __device__ float __ocml_modf_f32(float, __PRIVATE_AS float *);
-__device__ __attribute__((const)) float __ocml_nearbyint_f32(float);
 __device__ __attribute__((const)) float __ocml_nextafter_f32(float, float);
 __device__ __attribute__((const)) float __ocml_len3_f32(float, float, float);
 __device__ __attribute__((const)) float __ocml_len4_f32(float, float, float,
@@ -89,15 +63,10 @@ __device__ __attribute__((pure)) float __ocml_rcbrt_f32(float);
 __device__ __attribute__((const)) float __ocml_remainder_f32(float, float);
 __device__ float __ocml_remquo_f32(float, float, __PRIVATE_AS int *);
 __device__ __attribute__((const)) float __ocml_rhypot_f32(float, float);
-__device__ __attribute__((const)) float __ocml_rint_f32(float);
 __device__ __attribute__((const)) float __ocml_rlen3_f32(float, float, float);
 __device__ __attribute__((const)) float __ocml_rlen4_f32(float, float, float,
                                                          float);
-__device__ __attribute__((const)) float __ocml_round_f32(float);
 __device__ __attribute__((pure)) float __ocml_rsqrt_f32(float);
-__device__ __attribute__((const)) float __ocml_scalb_f32(float, float);
-__device__ __attribute__((const)) float __ocml_scalbn_f32(float, int);
-__device__ __attribute__((const)) int __ocml_signbit_f32(float);
 __device__ float __ocml_sincos_f32(float, __PRIVATE_AS float *);
 __device__ float __ocml_sincospi_f32(float, __PRIVATE_AS float *);
 __device__ float __ocml_sin_f32(float);
@@ -109,7 +78,6 @@ __device__ __attribute__((const)) float __ocml_native_sqrt_f32(float);
 __device__ float __ocml_tan_f32(float);
 __device__ __attribute__((pure)) float __ocml_tanh_f32(float);
 __device__ float __ocml_tgamma_f32(float);
-__device__ __attribute__((const)) float __ocml_trunc_f32(float);
 __device__ float __ocml_y0_f32(float);
 __device__ float __ocml_y1_f32(float);
 
@@ -150,8 +118,6 @@ __device__ __attribute__((const)) double __ocml_atan2_f64(double, double);
 __device__ __attribute__((const)) double __ocml_atan_f64(double);
 __device__ __attribute__((pure)) double __ocml_atanh_f64(double);
 __device__ __attribute__((pure)) double __ocml_cbrt_f64(double);
-__device__ __attribute__((const)) double __ocml_ceil_f64(double);
-__device__ __attribute__((const)) double __ocml_copysign_f64(double, double);
 __device__ double __ocml_cos_f64(double);
 __device__ __attribute__((pure)) double __ocml_cosh_f64(double);
 __device__ double __ocml_cospi_f64(double);
@@ -166,30 +132,18 @@ __device__ __attribute__((pure)) double __ocml_exp10_f64(double);
 __device__ __attribute__((pure)) double __ocml_exp2_f64(double);
 __device__ __attribute__((pure)) double __ocml_exp_f64(double);
 __device__ __attribute__((pure)) double __ocml_expm1_f64(double);
-__device__ __attribute__((const)) double __ocml_fabs_f64(double);
 __device__ __attribute__((const)) double __ocml_fdim_f64(double, double);
-__device__ __attribute__((const)) double __ocml_floor_f64(double);
-__device__ __attribute__((const)) double __ocml_fma_f64(double, double, double);
-__device__ __attribute__((const)) double __ocml_fmax_f64(double, double);
-__device__ __attribute__((const)) double __ocml_fmin_f64(double, double);
 __device__ __attribute__((const)) double __ocml_fmod_f64(double, double);
-__device__ double __ocml_frexp_f64(double, __PRIVATE_AS int *);
 __device__ __attribute__((const)) double __ocml_hypot_f64(double, double);
 __device__ __attribute__((const)) int __ocml_ilogb_f64(double);
-__device__ __attribute__((const)) int __ocml_isfinite_f64(double);
-__device__ __attribute__((const)) int __ocml_isinf_f64(double);
-__device__ __attribute__((const)) int __ocml_isnan_f64(double);
 __device__ double __ocml_j0_f64(double);
 __device__ double __ocml_j1_f64(double);
-__device__ __attribute__((const)) double __ocml_ldexp_f64(double, int);
 __device__ double __ocml_lgamma_f64(double);
 __device__ __attribute__((pure)) double __ocml_log10_f64(double);
 __device__ __attribute__((pure)) double __ocml_log1p_f64(double);
 __device__ __attribute__((pure)) double __ocml_log2_f64(double);
-__device__ __attribute__((const)) double __ocml_logb_f64(double);
 __device__ __attribute__((pure)) double __ocml_log_f64(double);
 __device__ double __ocml_modf_f64(double, __PRIVATE_AS double *);
-__device__ __attribute__((const)) double __ocml_nearbyint_f64(double);
 __device__ __attribute__((const)) double __ocml_nextafter_f64(double, double);
 __device__ __attribute__((const)) double __ocml_len3_f64(double, double,
                                                          double);
@@ -203,26 +157,19 @@ __device__ __attribute__((pure)) double __ocml_rcbrt_f64(double);
 __device__ __attribute__((const)) double __ocml_remainder_f64(double, double);
 __device__ double __ocml_remquo_f64(double, double, __PRIVATE_AS int *);
 __device__ __attribute__((const)) double __ocml_rhypot_f64(double, double);
-__device__ __attribute__((const)) double __ocml_rint_f64(double);
 __device__ __attribute__((const)) double __ocml_rlen3_f64(double, double,
                                                           double);
 __device__ __attribute__((const)) double __ocml_rlen4_f64(double, double,
                                                           double, double);
-__device__ __attribute__((const)) double __ocml_round_f64(double);
 __device__ __attribute__((pure)) double __ocml_rsqrt_f64(double);
-__device__ __attribute__((const)) double __ocml_scalb_f64(double, double);
-__device__ __attribute__((const)) double __ocml_scalbn_f64(double, int);
-__device__ __attribute__((const)) int __ocml_signbit_f64(double);
 __device__ double __ocml_sincos_f64(double, __PRIVATE_AS double *);
 __device__ double __ocml_sincospi_f64(double, __PRIVATE_AS double *);
 __device__ double __ocml_sin_f64(double);
 __device__ __attribute__((pure)) double __ocml_sinh_f64(double);
 __device__ double __ocml_sinpi_f64(double);
-__device__ __attribute__((const)) double __ocml_sqrt_f64(double);
 __device__ double __ocml_tan_f64(double);
 __device__ __attribute__((pure)) double __ocml_tanh_f64(double);
 __device__ double __ocml_tgamma_f64(double);
-__device__ __attribute__((const)) double __ocml_trunc_f64(double);
 __device__ double __ocml_y0_f64(double);
 __device__ double __ocml_y1_f64(double);
 
@@ -256,30 +203,12 @@ __device__ __attribute__((const)) double __ocml_fma_rtp_f64(double, double,
 __device__ __attribute__((const)) double __ocml_fma_rtz_f64(double, double,
                                                             double);
 
-__device__ __attribute__((const)) _Float16 __ocml_ceil_f16(_Float16);
 __device__ _Float16 __ocml_cos_f16(_Float16);
 __device__ __attribute__((const)) _Float16 __ocml_cvtrtn_f16_f32(float);
 __device__ __attribute__((const)) _Float16 __ocml_cvtrtp_f16_f32(float);
 __device__ __attribute__((const)) _Float16 __ocml_cvtrtz_f16_f32(float);
-__device__ __attribute__((pure)) _Float16 __ocml_exp_f16(_Float16);
-__device__ __attribute__((pure)) _Float16 __ocml_exp10_f16(_Float16);
-__device__ __attribute__((pure)) _Float16 __ocml_exp2_f16(_Float16);
-__device__ __attribute__((const)) _Float16 __ocml_floor_f16(_Float16);
-__device__ __attribute__((const)) _Float16 __ocml_fma_f16(_Float16, _Float16,
-                                                          _Float16);
-__device__ __attribute__((const)) _Float16 __ocml_fmax_f16(_Float16, _Float16);
-__device__ __attribute__((const)) _Float16 __ocml_fmin_f16(_Float16, _Float16);
-__device__ __attribute__((const)) _Float16 __ocml_fabs_f16(_Float16);
-__device__ __attribute__((const)) int __ocml_isinf_f16(_Float16);
-__device__ __attribute__((const)) int __ocml_isnan_f16(_Float16);
-__device__ __attribute__((pure)) _Float16 __ocml_log_f16(_Float16);
-__device__ __attribute__((pure)) _Float16 __ocml_log10_f16(_Float16);
-__device__ __attribute__((pure)) _Float16 __ocml_log2_f16(_Float16);
-__device__ __attribute__((const)) _Float16 __ocml_rint_f16(_Float16);
 __device__ __attribute__((const)) _Float16 __ocml_rsqrt_f16(_Float16);
 __device__ _Float16 __ocml_sin_f16(_Float16);
-__device__ __attribute__((const)) _Float16 __ocml_sqrt_f16(_Float16);
-__device__ __attribute__((const)) _Float16 __ocml_trunc_f16(_Float16);
 __device__ __attribute__((pure)) _Float16 __ocml_pown_f16(_Float16, int);
 
 typedef _Float16 __2f16 __attribute__((ext_vector_type(2)));
@@ -323,6 +252,16 @@ __device__ void __asan_unpoison_memory_region(const void *addr,
 __device__ int __asan_address_is_poisoned(const void *addr);
 __device__ void *__asan_region_is_poisoned(void *beg, __SIZE_TYPE__ size);
 
+__device__ __attribute__((pure)) _Float16 __ocml_exp_f16(_Float16);
+__device__ __attribute__((pure)) _Float16 __ocml_exp10_f16(_Float16);
+__device__ __attribute__((pure)) _Float16 __ocml_exp2_f16(_Float16);
+__device__ __attribute__((const)) _Float16 __ocml_fma_f16(_Float16, _Float16,
+                                                          _Float16);
+__device__ __attribute__((pure)) _Float16 __ocml_log_f16(_Float16);
+__device__ __attribute__((pure)) _Float16 __ocml_log10_f16(_Float16);
+__device__ __attribute__((pure)) _Float16 __ocml_log2_f16(_Float16);
+
+__device__ __attribute__((const)) _Float16 __ocml_sqrt_f16(_Float16);
 #if __has_feature(address_sanitizer)
 #define ASAN_POISON_MEMORY_REGION(addr, size)                                  \
   __asan_poison_memory_region((addr), (size))
diff --git a/clang/lib/Sema/AnalysisBasedWarnings.cpp b/clang/lib/Sema/AnalysisBasedWarnings.cpp
index 52fe7765c6c6a..e070d9f1a9b85 100644
--- a/clang/lib/Sema/AnalysisBasedWarnings.cpp
+++ b/clang/lib/Sema/AnalysisBasedWarnings.cpp
@@ -2929,7 +2929,9 @@ LifetimeSafetyTUAnalysis(Sema &S, TranslationUnitDecl *TU,
     AC.getCFGBuildOptions().AddCXXDefaultInitExprInCtors = true;
     AC.getCFGBuildOptions().setAllAlwaysAdd();
     if (AC.getCFG())
-      runLifetimeSafetyAnalysis(AC, &SemaHelper, LSStats, S.CollectStats);
+      runLifetimeSafetyAnalysis(AC, &SemaHelper,
+                                lifetimes::GetLifetimeSafetyOpts(S, FD),
+                                LSStats, S.CollectStats);
   }
 }
 
@@ -3159,13 +3161,12 @@ void clang::sema::AnalysisBasedWarnings::IssueWarnings(
     }
   }
 
-  // TODO: Enable lifetime safety analysis for other languages once it is
-  // stable.
   if (EnableLifetimeSafetyAnalysis) {
     if (AC.getCFG()) {
       lifetimes::LifetimeSafetySemaHelperImpl LifetimeSafetySemaHelper(S);
-      lifetimes::runLifetimeSafetyAnalysis(AC, &LifetimeSafetySemaHelper,
-                                           LSStats, S.CollectStats);
+      lifetimes::runLifetimeSafetyAnalysis(
+          AC, &LifetimeSafetySemaHelper, lifetimes::GetLifetimeSafetyOpts(S, D),
+          LSStats, S.CollectStats);
     }
   }
   // Check for violations of "called once" parameter properties.
diff --git a/clang/lib/Sema/SemaLifetimeSafety.h b/clang/lib/Sema/SemaLifetimeSafety.h
index 7c78a6a1e8c31..a3d546792fe41 100644
--- a/clang/lib/Sema/SemaLifetimeSafety.h
+++ b/clang/lib/Sema/SemaLifetimeSafety.h
@@ -76,6 +76,26 @@ inline bool IsLifetimeSafetyEnabled(Sema &S, const Decl *D) {
   return false;
 }
 
+inline bool ShouldSuggestLifetimeAnnotations(Sema &S, const Decl *D) {
+  DiagnosticsEngine &Diags = S.getDiagnostics();
+  constexpr unsigned DiagIDs[] = {
+      diag::warn_lifetime_safety_intra_tu_param_suggestion,
+      diag::warn_lifetime_safety_cross_tu_param_suggestion,
+      diag::warn_lifetime_safety_intra_tu_this_suggestion,
+      diag::warn_lifetime_safety_cross_tu_this_suggestion};
+  for (unsigned DiagID : DiagIDs)
+    if (!Diags.isIgnored(DiagID, D->getBeginLoc()))
+      return true;
+  return false;
+}
+
+inline LifetimeSafetyOpts GetLifetimeSafetyOpts(Sema &S, const Decl *D) {
+  LifetimeSafetyOpts LSOpts;
+  LSOpts.MaxCFGBlocks = S.getLangOpts().LifetimeSafetyMaxCFGBlocks;
+  LSOpts.SuggestAnnotations = ShouldSuggestLifetimeAnnotations(S, D);
+  return LSOpts;
+}
+
 class LifetimeSafetySemaHelperImpl : public LifetimeSafetySemaHelper {
 
 public:
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index d2bb312feadc1..7d9fff1051068 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -9109,10 +9109,13 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type,
       // it it breaks large amounts of Linux software.
       attr.setUsedAsTypeAttr();
       break;
-    case ParsedAttr::AT_OpenCLPrivateAddressSpace:
-    case ParsedAttr::AT_OpenCLGlobalAddressSpace:
     case ParsedAttr::AT_OpenCLGlobalDeviceAddressSpace:
     case ParsedAttr::AT_OpenCLGlobalHostAddressSpace:
+      state.getSema().Diag(attr.getLoc(), diag::warn_deprecated_attribute)
+          << attr;
+      [[fallthrough]];
+    case ParsedAttr::AT_OpenCLPrivateAddressSpace:
+    case ParsedAttr::AT_OpenCLGlobalAddressSpace:
     case ParsedAttr::AT_OpenCLLocalAddressSpace:
     case ParsedAttr::AT_OpenCLConstantAddressSpace:
     case ParsedAttr::AT_OpenCLGenericAddressSpace:
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp
index 4cbcaa2721639..97c655c103b0a 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp
@@ -384,7 +384,7 @@ void ExprEngine::processCallExit(ExplodedNode *CEBNode) {
 
     ExplodedNode *CEENode = Engine.makeNode(Loc, CEEState, N);
     if (!CEENode)
-      return;
+      continue;
 
     // Step 5: Perform the post-condition check of the CallExpr and enqueue the
     // result onto the work list.
diff --git a/clang/test/AST/ByteCode/evaluate-dtor.cpp b/clang/test/AST/ByteCode/evaluate-dtor.cpp
new file mode 100644
index 0000000000000..da0d1c2d4cafb
--- /dev/null
+++ b/clang/test/AST/ByteCode/evaluate-dtor.cpp
@@ -0,0 +1,56 @@
+// RUN: %clang_cc1 -std=c++23 -verify=both,expected %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++23 -verify=both,ref      %s
+
+struct MutableConst {
+  struct HasConstMember {
+    const int n = 4;
+  };
+  mutable HasConstMember hcm;
+  constexpr ~MutableConst() {
+    /// This is _not_ a read.
+    auto *p = &hcm.n;
+  }
+};
+constexpr MutableConst mc;
+
+
+struct  C {
+  static int m;
+  bool b;
+  constexpr C(bool b) : b(b){
+  }
+  constexpr ~C() {
+    if (b)
+      m++; // both-note {{a constant expression cannot modify an object that is visible outside that expression}}
+  }
+};
+
+int C::m = 0;
+constexpr C c1(false);
+constexpr C c2(true); // both-error {{must have constant destruction}} \
+                      // both-note {{in call to}}
+
+
+
+namespace LocalVariables {
+  struct S {
+    constexpr ~S() {
+      int a = 0;
+      ++a;
+    }
+  };
+  constexpr S s;
+}
+
+namespace GlobalVariables {
+  struct S {};
+  constexpr S s;
+
+  struct K {
+    constexpr ~K() {
+      s.~S(); // both-note {{a constant expression cannot modify an object that is visible outside that expression}}
+    }
+  };
+  constexpr K k{}; // both-error {{must have constant destruction}} \
+                   // both-note {{in call to}}
+}
diff --git a/clang/test/CIR/CodeGen/new-array-init.cpp b/clang/test/CIR/CodeGen/new-array-init.cpp
new file mode 100644
index 0000000000000..a44131aad87a6
--- /dev/null
+++ b/clang/test/CIR/CodeGen/new-array-init.cpp
@@ -0,0 +1,75 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --check-prefixes=LLVM,LLVMCIR --input-file=%t-cir.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --check-prefixes=LLVM,OGCG --input-file=%t.ll %s
+
+struct S {
+  int a;
+  int b;
+};
+
+// Value-init array new of a trivial type: () means zero-init via memset.
+S *makeVar(unsigned n) { return new S[n](); }
+
+// CIR-LABEL: cir.func{{.*}}@_Z7makeVarj
+// CIR:   cir.call @_Znam(
+// CIR:   cir.libc.memset {{.*}} bytes at {{.*}} to
+
+// LLVM-LABEL: @_Z7makeVarj
+// LLVM:   call {{.*}} ptr @_Znam(
+// LLVM:   call void @llvm.memset.p0.i64(ptr{{.*}} %{{.*}}, i8 0, i64 %{{.*}}, i1 false)
+
+// Constant element count: the size is folded to a constant.
+S *makeConst() { return new S[4](); }
+
+// CIR-LABEL: cir.func{{.*}}@_Z9makeConstv
+// CIR:   cir.call @_Znam(
+// CIR:   cir.libc.memset
+
+// LLVM-LABEL: @_Z9makeConstv
+// LLVM:   call {{.*}} ptr @_Znam(i64 noundef 32)
+// LLVM:   call void @llvm.memset.p0.i64(ptr{{.*}} %{{.*}}, i8 0, i64 32, i1 false)
+
+// No parens: default-init of a trivial type is a no-op (no memset).
+S *makeNoInit(unsigned n) { return new S[n]; }
+
+// CIR-LABEL: cir.func{{.*}}@_Z10makeNoInitj
+// CIR:   cir.call @_Znam(
+// CIR-NOT: cir.libc.memset
+
+// LLVM-LABEL: @_Z10makeNoInitj
+// LLVM:   call {{.*}} ptr @_Znam(
+// LLVM-NOT: memset
+
+// Braced-empty value-init goes through the InitListExpr path (also memset).
+S *makeBraced(unsigned n) { return new S[n]{}; }
+
+// CIR-LABEL: cir.func{{.*}}@_Z10makeBracedj
+// CIR:   cir.call @_Znam(
+// CIR:   cir.libc.memset
+
+// LLVM-LABEL: @_Z10makeBracedj
+// LLVM:   call {{.*}} ptr @_Znam(
+// LLVM:   call void @llvm.memset.p0.i64(ptr{{.*}} %{{.*}}, i8 0, i64 %{{.*}}, i1 false)
+
+// Non-zero-initializable element (pointer to data member): the null member
+// representation is -1, not 0, so memset is not used; the constructor-loop
+// path value-initializes each element to {0, -1}.
+struct M {
+  int x;
+  int M::*p;
+};
+
+M *makeMember(unsigned n) { return new M[n](); }
+
+// CIR-LABEL: cir.func{{.*}}@_Z10makeMemberj
+// CIR:   cir.call @_Znam(
+// CIR-NOT: cir.libc.memset
+// CIR:   cir.const #cir.const_record<{#cir.int<0> : !s32i, #cir.int<-1> : !s64i}>
+
+// LLVM-LABEL: @_Z10makeMemberj
+// LLVM:   call {{.*}} ptr @_Znam(
+// LLVMCIR: store %struct.M { i32 0, i64 -1 }, ptr %{{.*}}
+// OGCG:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{.*}}, ptr align 16 @{{.*}}
diff --git a/clang/test/CIR/CodeGen/new.cpp b/clang/test/CIR/CodeGen/new.cpp
index b198fb1e3d0d0..56f2d1f23179a 100644
--- a/clang/test/CIR/CodeGen/new.cpp
+++ b/clang/test/CIR/CodeGen/new.cpp
@@ -355,11 +355,11 @@ void t_constant_size_memset_init() {
 // CHECK:    %[[ELEM_PTR:.*]] = cir.cast bitcast %[[ALLOC_PTR]] : !cir.ptr<!void> -> !cir.ptr<!s32i>
 // CHECK:    %[[VOID_PTR:.*]] = cir.cast bitcast %[[ELEM_PTR]] : !cir.ptr<!s32i> -> !cir.ptr<!void>
 // CHECK:    %[[ZERO:.*]] = cir.const #cir.int<0> : !u8i
-// CHECK:    cir.libc.memset %[[ALLOCATION_SIZE]] bytes at %[[VOID_PTR]] to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
+// CHECK:    cir.libc.memset %[[ALLOCATION_SIZE]] bytes at %[[VOID_PTR]]{{.*}} to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
 
 // LLVM: define {{.*}} void @_Z27t_constant_size_memset_initv()
 // LLVM:   %[[P:.*]] = call noundef nonnull ptr @_Znam(i64 noundef 64)
-// LLVM:   call void @llvm.memset.p0.i64(ptr %[[P]], i8 0, i64 64, i1 false)
+// LLVM:   call void @llvm.memset.p0.i64(ptr{{.*}} %[[P]], i8 0, i64 64, i1 false)
 
 // OGCG: define {{.*}} void @_Z27t_constant_size_memset_initv()
 // OGCG:   %[[P:.*]] = call{{.*}} ptr @_Znam(i64{{.*}} 64)
@@ -432,7 +432,7 @@ void t_constant_size_partial_init() {
 // CHECK:    %[[REMAINING_SIZE:.*]] = cir.sub %[[ALLOCATION_SIZE]], %[[INIT_SIZE]] : !u64i
 // CHECK:    %[[VOID_PTR:.*]] = cir.cast bitcast %[[ELEM_3_PTR]] : !cir.ptr<!s32i> -> !cir.ptr<!void>
 // CHECK:    %[[ZERO:.*]] = cir.const #cir.int<0> : !u8i
-// CHECK:    cir.libc.memset %[[REMAINING_SIZE]] bytes at %[[VOID_PTR]] to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
+// CHECK:    cir.libc.memset %[[REMAINING_SIZE]] bytes at %[[VOID_PTR]]{{.*}} to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
 
 // LLVM: define {{.*}} void @_Z28t_constant_size_partial_initv()
 // LLVM:   %[[P:.*]] = call noundef nonnull ptr @_Znam(i64 {{.*}} 64)
@@ -442,7 +442,7 @@ void t_constant_size_partial_init() {
 // LLVM:   %[[ELEM_2:.*]] = getelementptr i32, ptr %[[ELEM_1]], i64 1
 // LLVM:   store i32 3, ptr %[[ELEM_2]]
 // LLVM:   %[[ELEM_3:.*]] = getelementptr i32, ptr %[[ELEM_2]], i64 1
-// LLVM:   call void @llvm.memset.p0.i64(ptr %[[ELEM_3]], i8 0, i64 52, i1 false)
+// LLVM:   call void @llvm.memset.p0.i64(ptr{{.*}} %[[ELEM_3]], i8 0, i64 52, i1 false)
 
 // OGCG: define {{.*}} void @_Z28t_constant_size_partial_initv()
 // OGCG:   %[[P:.*]] = call{{.*}} ptr @_Znam(i64{{.*}} 64)
@@ -614,7 +614,7 @@ void t_new_var_size6(int n) {
 // CHECK:    %[[REMAINING_SIZE:.*]] = cir.sub %[[ALLOC_SIZE]], %[[INIT_SIZE]] : !u64i
 // CHECK:    %[[PTR_DOUBLE_3_VOID:.*]] = cir.cast bitcast %[[PTR_DOUBLE_3]] : !cir.ptr<!cir.double> -> !cir.ptr<!void>
 // CHECK:    %[[ZERO:.*]] = cir.const #cir.int<0> : !u8i
-// CHECK:    cir.libc.memset{{.*}} bytes at %[[PTR_DOUBLE_3_VOID]] to %[[ZERO]]
+// CHECK:    cir.libc.memset{{.*}} bytes at %[[PTR_DOUBLE_3_VOID]]{{.*}} to %[[ZERO]]
 
 // LLVM: define{{.*}} void @_Z15t_new_var_size6i
 // LLVM:   %[[N:.*]] = load i32, ptr %{{.+}}
@@ -633,7 +633,7 @@ void t_new_var_size6(int n) {
 // LLVM:   store double 3.000000e+00, ptr %[[ELEM_2]], align 8
 // LLVM:   %[[ELEM_3:.*]] = getelementptr double, ptr %[[ELEM_2]], i64 1
 // LLVM:   %[[REMAINING_SIZE:.*]] = sub i64 %[[ALLOC_SIZE]], 24
-// LLVM:   call void @llvm.memset.p0.i64(ptr %[[ELEM_3]], i8 0, i64 %[[REMAINING_SIZE]], i1 false)
+// LLVM:   call void @llvm.memset.p0.i64(ptr{{.*}} %[[ELEM_3]], i8 0, i64 %[[REMAINING_SIZE]], i1 false)
 
 // OGCG: define{{.*}} void @_Z15t_new_var_size6i
 // OGCG:   %[[N:.*]] = load i32, ptr %{{.+}}
diff --git a/clang/test/CIR/CodeGen/paren-list-agg-init.cpp b/clang/test/CIR/CodeGen/paren-list-agg-init.cpp
index 01e045f376a4f..2b64dd60af9df 100644
--- a/clang/test/CIR/CodeGen/paren-list-agg-init.cpp
+++ b/clang/test/CIR/CodeGen/paren-list-agg-init.cpp
@@ -875,7 +875,7 @@ namespace gh68198 {
   // CIR: %[[ALLOC_DIFF:.*]] = cir.sub %[[SIZE]], %[[INIT_SIZE]] : !u64i
   // CIR: %[[ELT2_DECAY:.*]] = cir.cast bitcast %[[ELT2]] : !cir.ptr<!cir.array<!s32i x 2>> -> !cir.ptr<!void>
   // CIR: %[[ZERO:.*]] = cir.const #cir.int<0> : !u8i
-  // CIR: cir.libc.memset %[[ALLOC_DIFF]] bytes at %[[ELT2_DECAY]] to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
+  // CIR: cir.libc.memset %[[ALLOC_DIFF]] bytes at %[[ELT2_DECAY]]{{.*}} to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
   // CIR: %[[ARR_TO_VOID:.*]] = cir.cast bitcast %[[ALLOC_TO_ARR]] : !cir.ptr<!cir.array<!s32i x 2>> -> !cir.ptr<!void>
   // CIR: cir.store{{.*}} %[[ARR_TO_VOID]], %[[ARR_ALLOCA]] : !cir.ptr<!void>, !cir.ptr<!cir.ptr<!void>>
   void foo27() {
diff --git a/clang/test/CIR/CodeGenCXX/new-array-init.cpp b/clang/test/CIR/CodeGenCXX/new-array-init.cpp
index 6e94b8bc275d7..fd595351573cd 100644
--- a/clang/test/CIR/CodeGenCXX/new-array-init.cpp
+++ b/clang/test/CIR/CodeGenCXX/new-array-init.cpp
@@ -45,7 +45,7 @@ void fn(int n) {
   // CIR: %[[REST_SIZE:.*]] = cir.sub %[[ADJ_SIZE]], %[[TWELVE]] : !u64i
   // CIR: %[[REST_ALLOC_AS_VOID:.*]] = cir.cast bitcast %[[NEXT_ELT3]] : !cir.ptr<!s32i> -> !cir.ptr<!void>
   // CIR: %[[ZERO:.*]] = cir.const #cir.int<0> : !u8i
-  // CIR: cir.libc.memset %[[REST_SIZE]] bytes at %[[REST_ALLOC_AS_VOID]] to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
+  // CIR: cir.libc.memset %[[REST_SIZE]] bytes at %[[REST_ALLOC_AS_VOID]]{{.*}} to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
 
   // LLVM: %[[N_ALLOCA:.*]] = alloca i32
   // LLVM: %[[N_LOAD:.*]] = load i32, ptr %[[N_ALLOCA]]
@@ -100,7 +100,7 @@ void fn_paren(int n) {
   // CIR: %[[REST_SIZE:.*]] = cir.sub %[[ADJ_SIZE]], %[[TWELVE]] : !u64i
   // CIR: %[[REST_ALLOC_AS_VOID:.*]] = cir.cast bitcast %[[NEXT_ELT3]] : !cir.ptr<!s32i> -> !cir.ptr<!void>
   // CIR: %[[ZERO:.*]] = cir.const #cir.int<0> : !u8i
-  // CIR: cir.libc.memset %[[REST_SIZE]] bytes at %[[REST_ALLOC_AS_VOID]] to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
+  // CIR: cir.libc.memset %[[REST_SIZE]] bytes at %[[REST_ALLOC_AS_VOID]]{{.*}} to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
 
   // LLVM: %[[N_ALLOCA:.*]] = alloca i32
   // LLVM: %[[N_LOAD:.*]] = load i32, ptr %[[N_ALLOCA]]
@@ -206,7 +206,7 @@ void const_sufficient() {
   // CIR: %[[REST_SIZE:.*]] = cir.sub %[[SIZE]], %[[INIT_SIZE]] : !u64i
   // CIR: %[[REST_PTR_DECAY:.*]] = cir.cast bitcast %[[ELT3]] : !cir.ptr<!s32i> -> !cir.ptr<!void>
   // CIR: %[[ZERO:.*]] = cir.const #cir.int<0> : !u8i
-  // CIR: cir.libc.memset %[[REST_SIZE]] bytes at %[[REST_PTR_DECAY]] to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
+  // CIR: cir.libc.memset %[[REST_SIZE]] bytes at %[[REST_PTR_DECAY]]{{.*}} to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
   // CIR: cir.return
 
   // LLVM: %[[ALLOC:.*]] = call{{.*}}nonnull ptr @_Znam(i64 noundef 16)
@@ -243,7 +243,7 @@ void const_sufficient_paren() {
   // CIR: %[[REST_SIZE:.*]] = cir.sub %[[SIZE]], %[[INIT_SIZE]] : !u64i
   // CIR: %[[REST_PTR_DECAY:.*]] = cir.cast bitcast %[[ELT3]] : !cir.ptr<!s32i> -> !cir.ptr<!void>
   // CIR: %[[ZERO:.*]] = cir.const #cir.int<0> : !u8i
-  // CIR: cir.libc.memset %[[REST_SIZE]] bytes at %[[REST_PTR_DECAY]] to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
+  // CIR: cir.libc.memset %[[REST_SIZE]] bytes at %[[REST_PTR_DECAY]]{{.*}} to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
   // CIR: cir.return
 
   // LLVM: %[[ALLOC:.*]] = call{{.*}}nonnull ptr @_Znam(i64 noundef 16)
@@ -298,7 +298,7 @@ void string_nonconst(int n) {
   // CIR: %[[SIZE_LEFT:.*]] = cir.sub %[[SIZE]], %[[CONST_STR_SIZE]] : !u64i
   // CIR: %[[AFTER_COPY_CAST:.*]] = cir.cast bitcast %[[AFTER_COPY]] : !cir.ptr<!s8i> -> !cir.ptr<!void>
   // CIR: %[[ZERO:.*]] = cir.const #cir.int<0> : !u8i
-  // CIR: cir.libc.memset %[[SIZE_LEFT]] bytes at %[[AFTER_COPY_CAST]] to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
+  // CIR: cir.libc.memset %[[SIZE_LEFT]] bytes at %[[AFTER_COPY_CAST]]{{.*}} to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
 
   // LLVM: %[[ARG_ALLOCA:.*]] = alloca i32
   // LLVM: %[[ARG_LOAD:.*]] = load i32, ptr %[[ARG_ALLOCA]]
@@ -334,7 +334,7 @@ void string_nonconst_paren(int n) {
   // CIR: %[[SIZE_LEFT:.*]] = cir.sub %[[SIZE]], %[[CONST_STR_SIZE]] : !u64i
   // CIR: %[[AFTER_COPY_CAST:.*]] = cir.cast bitcast %[[AFTER_COPY]] : !cir.ptr<!s8i> -> !cir.ptr<!void>
   // CIR: %[[ZERO:.*]] = cir.const #cir.int<0> : !u8i
-  // CIR: cir.libc.memset %[[SIZE_LEFT]] bytes at %[[AFTER_COPY_CAST]] to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
+  // CIR: cir.libc.memset %[[SIZE_LEFT]] bytes at %[[AFTER_COPY_CAST]]{{.*}} to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
 
   // LLVM: %[[ARG_ALLOCA:.*]] = alloca i32
   // LLVM: %[[ARG_LOAD:.*]] = load i32, ptr %[[ARG_ALLOCA]]
@@ -370,7 +370,7 @@ void string_nonconst_paren_extra_paren(int n) {
   // CIR: %[[SIZE_LEFT:.*]] = cir.sub %[[SIZE]], %[[CONST_STR_SIZE]] : !u64i
   // CIR: %[[AFTER_COPY_CAST:.*]] = cir.cast bitcast %[[AFTER_COPY]] : !cir.ptr<!s8i> -> !cir.ptr<!void>
   // CIR: %[[ZERO:.*]] = cir.const #cir.int<0> : !u8i
-  // CIR: cir.libc.memset %[[SIZE_LEFT]] bytes at %[[AFTER_COPY_CAST]] to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
+  // CIR: cir.libc.memset %[[SIZE_LEFT]] bytes at %[[AFTER_COPY_CAST]]{{.*}} to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
 
   // LLVM: %[[ARG_ALLOCA:.*]] = alloca i32
   // LLVM: %[[ARG_LOAD:.*]] = load i32, ptr %[[ARG_ALLOCA]]
@@ -540,7 +540,7 @@ void aggr_sufficient(int n) {
   // CIR: %[[REST_SIZE:.*]] = cir.sub %[[SIZE]], %[[TWO_ELTS_SIZE]] : !u64i
   // CIR: %[[REST_DECAY:.*]] = cir.cast bitcast %[[ELT2]] : !cir.ptr<!rec_Aggr2E0> -> !cir.ptr<!void>
   // CIR: %[[ZERO:.*]] = cir.const #cir.int<0> : !u8i
-  // CIR: cir.libc.memset %[[REST_SIZE]] bytes at %[[REST_DECAY]] to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
+  // CIR: cir.libc.memset %[[REST_SIZE]] bytes at %[[REST_DECAY]]{{.*}} to %[[ZERO]] : !cir.ptr<!void>, !u8i, !u64i
 
   // LLVM: %[[ARG:.*]] = alloca i32
   // LLVM: %[[GET_N:.*]] = load i32, ptr %[[ARG]]
diff --git a/clang/test/CIR/CodeGenOpenMP/not-yet-implemented.c b/clang/test/CIR/CodeGenOpenMP/not-yet-implemented.c
index 6d59f45d6e5e4..29e280bf262d0 100644
--- a/clang/test/CIR/CodeGenOpenMP/not-yet-implemented.c
+++ b/clang/test/CIR/CodeGenOpenMP/not-yet-implemented.c
@@ -10,9 +10,7 @@ void do_things() {
   {}
 
   int i;
-  // TODO(OMP): We might consider overloading operator<< for OMPClauseKind in
-  // the future if we want to improve this.
-  // expected-error@+1{{ClangIR code gen Not Yet Implemented: OpenMPClause : if}}
+  // expected-error@+1{{ClangIR code gen Not Yet Implemented: OpenMP PARALLEL 'if' clause}}
 #pragma omp parallel if(i)
   {}
 }
diff --git a/clang/test/CIR/CodeGenOpenMP/target-map-llvm-device.c b/clang/test/CIR/CodeGenOpenMP/target-map-llvm-device.c
new file mode 100644
index 0000000000000..b6fa078d143c5
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenMP/target-map-llvm-device.c
@@ -0,0 +1,130 @@
+// Two-step host-BC ->  device pipeline that mirrors the offloading driver.
+//
+// Step 1: Host compilation to bitcode (provides offload entry info to device pass).
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa \
+// RUN:   -fclangir -emit-llvm-bc %s -o %t-cir-host.bc
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa \
+// RUN:   -emit-llvm-bc %s -o %t-ogcg-host.bc
+//
+// Step 2: Device compilation using host BC.
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-target-device \
+// RUN:   -fopenmp-host-ir-file-path %t-cir-host.bc \
+// RUN:   -fclangir -emit-llvm %s -o - \
+// RUN:   | FileCheck %s --check-prefix=LLVM
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-target-device \
+// RUN:   -fopenmp-host-ir-file-path %t-ogcg-host.bc \
+// RUN:   -emit-llvm %s -o - \
+// RUN:   | FileCheck %s --check-prefix=OGCG
+
+void use(int);
+
+void target_map_to(int x) {
+#pragma omp target map(to : x)
+  {
+    use(x);
+  }
+}
+
+void target_map_from(int x) {
+#pragma omp target map(from : x)
+  {
+    x = 42;
+  }
+}
+
+void target_map_tofrom(int x) {
+#pragma omp target map(tofrom : x)
+  {
+    x = x + 1;
+  }
+}
+
+void target_map_multiple(int a, int b) {
+#pragma omp target map(to : a) map(from : b)
+  {
+    b = a;
+  }
+}
+
+// LLVM-LABEL: define weak_odr protected amdgpu_kernel void @__omp_offloading_{{.*}}_target_map_to_l
+// LLVM-SAME:  (ptr %[[ARG:[^,]+]], ptr
+// LLVM:         %[[SLOT:.*]] = addrspacecast ptr addrspace(5) %{{.*}} to ptr
+// LLVM:         store ptr %[[ARG]], ptr %[[SLOT]], align 8
+// LLVM:         call i32 @__kmpc_target_init(
+// LLVM:       user_code.entry:
+// LLVM:         %[[PTR:.*]] = load ptr, ptr %[[SLOT]], align 8
+// LLVM:         %[[V:.*]] = load i32, ptr %[[PTR]], align 4
+// LLVM:         call void @use(i32 {{.*}} %[[V]])
+// LLVM:         call void @__kmpc_target_deinit()
+// LLVM:         ret void
+
+// LLVM-LABEL: define weak_odr protected amdgpu_kernel void @__omp_offloading_{{.*}}_target_map_from_l
+// LLVM-SAME:  (ptr %[[ARG:[^,]+]], ptr
+// LLVM:         %[[SLOT:.*]] = addrspacecast ptr addrspace(5) %{{.*}} to ptr
+// LLVM:         store ptr %[[ARG]], ptr %[[SLOT]], align 8
+// LLVM:         call i32 @__kmpc_target_init(
+// LLVM:       user_code.entry:
+// LLVM:         %[[PTR:.*]] = load ptr, ptr %[[SLOT]], align 8
+// LLVM:         store i32 42, ptr %[[PTR]], align 4
+// LLVM:         call void @__kmpc_target_deinit()
+// LLVM:         ret void
+
+// LLVM-LABEL: define weak_odr protected amdgpu_kernel void @__omp_offloading_{{.*}}_target_map_tofrom_l
+// LLVM-SAME:  (ptr %[[ARG:[^,]+]], ptr
+// LLVM:         %[[SLOT:.*]] = addrspacecast ptr addrspace(5) %{{.*}} to ptr
+// LLVM:         store ptr %[[ARG]], ptr %[[SLOT]], align 8
+// LLVM:         call i32 @__kmpc_target_init(
+// LLVM:       user_code.entry:
+// LLVM:         %[[PTR:.*]] = load ptr, ptr %[[SLOT]], align 8
+// LLVM:         %[[LD:.*]] = load i32, ptr %[[PTR]], align 4
+// LLVM:         %[[ADD:.*]] = add nsw i32 %[[LD]], 1
+// LLVM:         store i32 %[[ADD]], ptr %[[PTR]], align 4
+// LLVM:         call void @__kmpc_target_deinit()
+// LLVM:         ret void
+
+// LLVM-LABEL: define weak_odr protected amdgpu_kernel void @__omp_offloading_{{.*}}_target_map_multiple_l
+// LLVM-SAME:  (ptr %[[ARG_A:[^,]+]], ptr %[[ARG_B:[^,]+]], ptr
+// LLVM:         %[[SLOT_A:.*]] = addrspacecast ptr addrspace(5) %{{.*}} to ptr
+// LLVM:         store ptr %[[ARG_A]], ptr %[[SLOT_A]], align 8
+// LLVM:         %[[SLOT_B:.*]] = addrspacecast ptr addrspace(5) %{{.*}} to ptr
+// LLVM:         store ptr %[[ARG_B]], ptr %[[SLOT_B]], align 8
+// LLVM:         call i32 @__kmpc_target_init(
+// LLVM:       user_code.entry:
+// LLVM:         %[[PTR_A:.*]] = load ptr, ptr %[[SLOT_A]], align 8
+// LLVM:         %[[PTR_B:.*]] = load ptr, ptr %[[SLOT_B]], align 8
+// LLVM:         %[[A:.*]] = load i32, ptr %[[PTR_A]], align 4
+// LLVM:         store i32 %[[A]], ptr %[[PTR_B]], align 4
+// LLVM:         call void @__kmpc_target_deinit()
+// LLVM:         ret void
+
+// OGCG-LABEL: define weak_odr protected amdgpu_kernel void @__omp_offloading_{{.*}}_target_map_to_l
+// OGCG:         call i32 @__kmpc_target_init(
+// OGCG:       user_code.entry:
+// OGCG:         %[[V:.*]] = load i32, ptr %{{.*}}, align 4
+// OGCG:         call void @use(i32 {{.*}} %[[V]])
+// OGCG:         call void @__kmpc_target_deinit()
+// OGCG:         ret void
+
+// OGCG-LABEL: define weak_odr protected amdgpu_kernel void @__omp_offloading_{{.*}}_target_map_from_l
+// OGCG:         call i32 @__kmpc_target_init(
+// OGCG:       user_code.entry:
+// OGCG:         store i32 42, ptr %{{.*}}, align 4
+// OGCG:         call void @__kmpc_target_deinit()
+// OGCG:         ret void
+
+// OGCG-LABEL: define weak_odr protected amdgpu_kernel void @__omp_offloading_{{.*}}_target_map_tofrom_l
+// OGCG:         call i32 @__kmpc_target_init(
+// OGCG:       user_code.entry:
+// OGCG:         %[[LD:.*]] = load i32, ptr %{{.*}}, align 4
+// OGCG:         %[[ADD:.*]] = add nsw i32 %[[LD]], 1
+// OGCG:         store i32 %[[ADD]], ptr %{{.*}}, align 4
+// OGCG:         call void @__kmpc_target_deinit()
+// OGCG:         ret void
+
+// OGCG-LABEL: define weak_odr protected amdgpu_kernel void @__omp_offloading_{{.*}}_target_map_multiple_l
+// OGCG:         call i32 @__kmpc_target_init(
+// OGCG:       user_code.entry:
+// OGCG:         %[[A:.*]] = load i32, ptr %{{.*}}, align 4
+// OGCG:         store i32 %[[A]], ptr %{{.*}}, align 4
+// OGCG:         call void @__kmpc_target_deinit()
+// OGCG:         ret void
diff --git a/clang/test/CIR/CodeGenOpenMP/target-map-llvm-host.c b/clang/test/CIR/CodeGenOpenMP/target-map-llvm-host.c
new file mode 100644
index 0000000000000..a42e955396980
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenMP/target-map-llvm-host.c
@@ -0,0 +1,163 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fopenmp \
+// RUN:   -fopenmp-targets=amdgcn-amd-amdhsa -fclangir -emit-llvm %s -o - \
+// RUN:   | FileCheck %s --check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fopenmp \
+// RUN:   -fopenmp-targets=amdgcn-amd-amdhsa -emit-llvm %s -o - \
+// RUN:   | FileCheck %s --check-prefix=OGCG
+
+void use(int);
+
+void target_map_to(int x) {
+#pragma omp target map(to : x)
+  {
+    use(x);
+  }
+}
+
+void target_map_from(int x) {
+#pragma omp target map(from : x)
+  {
+    x = 42;
+  }
+}
+
+void target_map_tofrom(int x) {
+#pragma omp target map(tofrom : x)
+  {
+    x = x + 1;
+  }
+}
+
+void target_map_multiple(int a, int b) {
+#pragma omp target map(to : a) map(from : b)
+  {
+    b = a;
+  }
+}
+
+// Host wrappers
+//
+// LLVM-LABEL: define {{.*}} void @target_map_to(
+// LLVM-SAME:  i32 noundef %[[ARG:[^,)]+]]
+// LLVM:         %[[X_ADDR:.*]] = alloca i32, i64 1, align 4
+// LLVM:         store i32 %[[ARG]], ptr %[[X_ADDR]], align 4
+// LLVM:         %[[BP:.*]] = getelementptr inbounds [2 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
+// LLVM:         store ptr %[[X_ADDR]], ptr %[[BP]], align 8
+// LLVM:         %[[P:.*]] = getelementptr inbounds [2 x ptr], ptr %.offload_ptrs, i32 0, i32 0
+// LLVM:         store ptr %[[X_ADDR]], ptr %[[P]], align 8
+// LLVM:         call i32 @__tgt_target_kernel(
+// LLVM:       omp_offload.failed:
+// LLVM:         call void @__omp_offloading_{{.*}}_target_map_to_l{{.*}}(ptr %[[X_ADDR]], ptr null)
+
+// LLVM-LABEL: define {{.*}} void @target_map_from(
+// LLVM-SAME:  i32 noundef %[[ARG:[^,)]+]]
+// LLVM:         %[[X_ADDR:.*]] = alloca i32, i64 1, align 4
+// LLVM:         store i32 %[[ARG]], ptr %[[X_ADDR]], align 4
+// LLVM:         %[[BP:.*]] = getelementptr inbounds [2 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
+// LLVM:         store ptr %[[X_ADDR]], ptr %[[BP]], align 8
+// LLVM:         %[[P:.*]] = getelementptr inbounds [2 x ptr], ptr %.offload_ptrs, i32 0, i32 0
+// LLVM:         store ptr %[[X_ADDR]], ptr %[[P]], align 8
+// LLVM:         call i32 @__tgt_target_kernel(
+// LLVM:       omp_offload.failed:
+// LLVM:         call void @__omp_offloading_{{.*}}_target_map_from_l{{.*}}(ptr %[[X_ADDR]], ptr null)
+
+// LLVM-LABEL: define {{.*}} void @target_map_tofrom(
+// LLVM-SAME:  i32 noundef %[[ARG:[^,)]+]]
+// LLVM:         %[[X_ADDR:.*]] = alloca i32, i64 1, align 4
+// LLVM:         store i32 %[[ARG]], ptr %[[X_ADDR]], align 4
+// LLVM:         %[[BP:.*]] = getelementptr inbounds [2 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
+// LLVM:         store ptr %[[X_ADDR]], ptr %[[BP]], align 8
+// LLVM:         %[[P:.*]] = getelementptr inbounds [2 x ptr], ptr %.offload_ptrs, i32 0, i32 0
+// LLVM:         store ptr %[[X_ADDR]], ptr %[[P]], align 8
+// LLVM:         call i32 @__tgt_target_kernel(
+// LLVM:       omp_offload.failed:
+// LLVM:         call void @__omp_offloading_{{.*}}_target_map_tofrom_l{{.*}}(ptr %[[X_ADDR]], ptr null)
+
+// LLVM-LABEL: define {{.*}} void @target_map_multiple(
+// LLVM-SAME:  i32 noundef %[[ARG_A:[^,)]+]], i32 noundef %[[ARG_B:[^,)]+]]
+// LLVM:         %[[A_ADDR:.*]] = alloca i32, i64 1, align 4
+// LLVM:         %[[B_ADDR:.*]] = alloca i32, i64 1, align 4
+// LLVM:         store i32 %[[ARG_A]], ptr %[[A_ADDR]], align 4
+// LLVM:         store i32 %[[ARG_B]], ptr %[[B_ADDR]], align 4
+// LLVM:         %[[BP_A:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
+// LLVM:         store ptr %[[A_ADDR]], ptr %[[BP_A]], align 8
+// LLVM:         %[[P_A:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_ptrs, i32 0, i32 0
+// LLVM:         store ptr %[[A_ADDR]], ptr %[[P_A]], align 8
+// LLVM:         %[[BP_B:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_baseptrs, i32 0, i32 1
+// LLVM:         store ptr %[[B_ADDR]], ptr %[[BP_B]], align 8
+// LLVM:         %[[P_B:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_ptrs, i32 0, i32 1
+// LLVM:         store ptr %[[B_ADDR]], ptr %[[P_B]], align 8
+// LLVM:         call i32 @__tgt_target_kernel(
+// LLVM:       omp_offload.failed:
+// LLVM:         call void @__omp_offloading_{{.*}}_target_map_multiple_l{{.*}}(ptr %[[A_ADDR]], ptr %[[B_ADDR]], ptr null)
+
+// Outlined target functions
+//
+// The mapped pointer arrives as the first function argument; load/store the
+// user value directly through it.
+
+// LLVM-LABEL: define internal void @__omp_offloading_{{.*}}_target_map_to_l
+// LLVM-SAME:  (ptr %[[ARG:[^,]+]], ptr
+// LLVM:         %[[V:.*]] = load i32, ptr %[[ARG]], align 4
+// LLVM:         call void @use(i32 {{.*}} %[[V]])
+// LLVM:         ret void
+
+// LLVM-LABEL: define internal void @__omp_offloading_{{.*}}_target_map_from_l
+// LLVM-SAME:  (ptr %[[ARG:[^,]+]], ptr
+// LLVM:         store i32 42, ptr %[[ARG]], align 4
+// LLVM:         ret void
+
+// LLVM-LABEL: define internal void @__omp_offloading_{{.*}}_target_map_tofrom_l
+// LLVM-SAME:  (ptr %[[ARG:[^,]+]], ptr
+// LLVM:         %[[LD:.*]] = load i32, ptr %[[ARG]], align 4
+// LLVM:         %[[ADD:.*]] = add nsw i32 %[[LD]], 1
+// LLVM:         store i32 %[[ADD]], ptr %[[ARG]], align 4
+// LLVM:         ret void
+
+// LLVM-LABEL: define internal void @__omp_offloading_{{.*}}_target_map_multiple_l
+// LLVM-SAME:  (ptr %[[ARG_A:[^,]+]], ptr %[[ARG_B:[^,]+]], ptr
+// LLVM:         %[[A:.*]] = load i32, ptr %[[ARG_A]], align 4
+// LLVM:         store i32 %[[A]], ptr %[[ARG_B]], align 4
+// LLVM:         ret void
+
+// OGCG interleaves host wrapper and outlined function per target region.
+
+// OGCG-LABEL: define {{.*}} void @target_map_to(i32
+// OGCG:         call i32 @__tgt_target_kernel(
+// OGCG:       omp_offload.failed:
+// OGCG:         call void @__omp_offloading_{{.*}}_target_map_to_l
+
+// OGCG-LABEL: define internal void @__omp_offloading_{{.*}}_target_map_to_l
+// OGCG:         %[[V:.*]] = load i32, ptr %{{.*}}, align 4
+// OGCG:         call void @use(i32 {{.*}} %[[V]])
+// OGCG:         ret void
+
+// OGCG-LABEL: define {{.*}} void @target_map_from(i32
+// OGCG:         call i32 @__tgt_target_kernel(
+// OGCG:       omp_offload.failed:
+// OGCG:         call void @__omp_offloading_{{.*}}_target_map_from_l
+
+// OGCG-LABEL: define internal void @__omp_offloading_{{.*}}_target_map_from_l
+// OGCG:         store i32 42, ptr %{{.*}}, align 4
+// OGCG:         ret void
+
+// OGCG-LABEL: define {{.*}} void @target_map_tofrom(i32
+// OGCG:         call i32 @__tgt_target_kernel(
+// OGCG:       omp_offload.failed:
+// OGCG:         call void @__omp_offloading_{{.*}}_target_map_tofrom_l
+
+// OGCG-LABEL: define internal void @__omp_offloading_{{.*}}_target_map_tofrom_l
+// OGCG:         %[[LD:.*]] = load i32, ptr %{{.*}}, align 4
+// OGCG:         %[[ADD:.*]] = add nsw i32 %[[LD]], 1
+// OGCG:         store i32 %[[ADD]], ptr %{{.*}}, align 4
+// OGCG:         ret void
+
+// OGCG-LABEL: define {{.*}} void @target_map_multiple(i32
+// OGCG:         call i32 @__tgt_target_kernel(
+// OGCG:       omp_offload.failed:
+// OGCG:         call void @__omp_offloading_{{.*}}_target_map_multiple_l
+
+// OGCG-LABEL: define internal void @__omp_offloading_{{.*}}_target_map_multiple_l
+// OGCG:         %[[A:.*]] = load i32, ptr %{{.*}}, align 4
+// OGCG:         store i32 %[[A]], ptr %{{.*}}, align 4
+// OGCG:         ret void
diff --git a/clang/test/CIR/CodeGenOpenMP/target-map.c b/clang/test/CIR/CodeGenOpenMP/target-map.c
new file mode 100644
index 0000000000000..394b7abb9e0b2
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenMP/target-map.c
@@ -0,0 +1,105 @@
+// Host compilation (x86 host, AMDGPU offload target): no address space on allocas.
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa -emit-cir -fclangir %s -o - \
+// RUN:   | FileCheck %s --check-prefix=CIR-HOST
+
+// Device compilation (AMDGPU): allocas in private address space, addrspacecast for map info.
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-target-device \
+// RUN:   -emit-cir -fclangir %s -o - \
+// RUN:   | FileCheck %s --check-prefix=CIR-DEVICE
+
+void use(int);
+
+void target_map_to(int x) {
+  // CIR-HOST: cir.func{{.*}}@target_map_to
+  // CIR-HOST: %[[X_ALLOCA:.*]] = cir.alloca "x" align(4) init : !cir.ptr<!s32i>
+  // CIR-HOST: %[[MAP:.*]] = omp.map.info var_ptr(%[[X_ALLOCA]] : !cir.ptr<!s32i>, !s32i) map_clauses(to) capture(ByRef) -> !cir.ptr<!s32i> {name = "x"}
+  // CIR-HOST-NEXT: omp.target kernel_type(generic) map_entries(%[[MAP]] -> %[[ARG:.*]] : !cir.ptr<!s32i>) {
+  // CIR-HOST-NEXT: %[[LOAD:.*]] = cir.load align(4) %[[ARG]]
+  // CIR-HOST-NEXT: cir.call @use(%[[LOAD]])
+  // CIR-HOST-NEXT: omp.terminator
+  // CIR-HOST-NEXT: }
+
+  // CIR-DEVICE: cir.func{{.*}}@target_map_to
+  // CIR-DEVICE: %[[X_ALLOCA:.*]] = cir.alloca "x" align(4) init : !cir.ptr<!s32i, target_address_space(5)>
+  // CIR-DEVICE: %[[CAST:.*]] = cir.cast address_space %[[X_ALLOCA]] : !cir.ptr<!s32i, target_address_space(5)> -> !cir.ptr<!s32i>
+  // CIR-DEVICE: %[[MAP:.*]] = omp.map.info var_ptr(%[[CAST]] : !cir.ptr<!s32i>, !s32i) map_clauses(to) capture(ByRef) -> !cir.ptr<!s32i> {name = "x"}
+  // CIR-DEVICE-NEXT: omp.target kernel_type(generic) map_entries(%[[MAP]] -> %[[ARG:.*]] : !cir.ptr<!s32i>) {
+  // CIR-DEVICE: omp.terminator
+  // CIR-DEVICE-NEXT: }
+#pragma omp target map(to : x)
+  {
+    use(x);
+  }
+}
+
+void target_map_from(int x) {
+  // CIR-HOST: cir.func{{.*}}@target_map_from
+  // CIR-HOST: %[[X_ALLOCA:.*]] = cir.alloca "x" align(4) init : !cir.ptr<!s32i>
+  // CIR-HOST: %[[MAP:.*]] = omp.map.info var_ptr(%[[X_ALLOCA]] : !cir.ptr<!s32i>, !s32i) map_clauses(from) capture(ByRef) -> !cir.ptr<!s32i> {name = "x"}
+  // CIR-HOST-NEXT: omp.target kernel_type(generic) map_entries(%[[MAP]] -> %[[ARG:.*]] : !cir.ptr<!s32i>) {
+  // CIR-HOST-NEXT: %[[C42:.*]] = cir.const #cir.int<42> : !s32i
+  // CIR-HOST-NEXT: cir.store align(4) %[[C42]], %[[ARG]]
+  // CIR-HOST-NEXT: omp.terminator
+  // CIR-HOST-NEXT: }
+
+  // CIR-DEVICE: cir.func{{.*}}@target_map_from
+  // CIR-DEVICE: %[[X_ALLOCA:.*]] = cir.alloca "x" align(4) init : !cir.ptr<!s32i, target_address_space(5)>
+  // CIR-DEVICE: %[[CAST:.*]] = cir.cast address_space %[[X_ALLOCA]] : !cir.ptr<!s32i, target_address_space(5)> -> !cir.ptr<!s32i>
+  // CIR-DEVICE: %[[MAP:.*]] = omp.map.info var_ptr(%[[CAST]] : !cir.ptr<!s32i>, !s32i) map_clauses(from) capture(ByRef) -> !cir.ptr<!s32i> {name = "x"}
+  // CIR-DEVICE-NEXT: omp.target kernel_type(generic) map_entries(%[[MAP]] -> %[[ARG:.*]] : !cir.ptr<!s32i>) {
+  // CIR-DEVICE: omp.terminator
+  // CIR-DEVICE-NEXT: }
+#pragma omp target map(from : x)
+  {
+    x = 42;
+  }
+}
+
+void target_map_tofrom(int x) {
+  // CIR-HOST: cir.func{{.*}}@target_map_tofrom
+  // CIR-HOST: %[[X_ALLOCA:.*]] = cir.alloca "x" align(4) init : !cir.ptr<!s32i>
+  // CIR-HOST: %[[MAP:.*]] = omp.map.info var_ptr(%[[X_ALLOCA]] : !cir.ptr<!s32i>, !s32i) map_clauses(tofrom) capture(ByRef) -> !cir.ptr<!s32i> {name = "x"}
+  // CIR-HOST-NEXT: omp.target kernel_type(generic) map_entries(%[[MAP]] -> %[[ARG:.*]] : !cir.ptr<!s32i>) {
+  // CIR-HOST: omp.terminator
+  // CIR-HOST-NEXT: }
+
+  // CIR-DEVICE: cir.func{{.*}}@target_map_tofrom
+  // CIR-DEVICE: %[[X_ALLOCA:.*]] = cir.alloca "x" align(4) init : !cir.ptr<!s32i, target_address_space(5)>
+  // CIR-DEVICE: %[[CAST:.*]] = cir.cast address_space %[[X_ALLOCA]] : !cir.ptr<!s32i, target_address_space(5)> -> !cir.ptr<!s32i>
+  // CIR-DEVICE: %[[MAP:.*]] = omp.map.info var_ptr(%[[CAST]] : !cir.ptr<!s32i>, !s32i) map_clauses(tofrom) capture(ByRef) -> !cir.ptr<!s32i> {name = "x"}
+  // CIR-DEVICE-NEXT: omp.target kernel_type(generic) map_entries(%[[MAP]] -> %[[ARG:.*]] : !cir.ptr<!s32i>) {
+  // CIR-DEVICE: omp.terminator
+  // CIR-DEVICE-NEXT: }
+#pragma omp target map(tofrom : x)
+  {
+    x = x + 1;
+  }
+}
+
+void target_map_multiple(int a, int b) {
+  // CIR-HOST: cir.func{{.*}}@target_map_multiple
+  // CIR-HOST-DAG: %[[A_ALLOCA:.*]] = cir.alloca "a" align(4) init : !cir.ptr<!s32i>
+  // CIR-HOST-DAG: %[[B_ALLOCA:.*]] = cir.alloca "b" align(4) init : !cir.ptr<!s32i>
+  // CIR-HOST: %[[MAP_A:.*]] = omp.map.info var_ptr(%[[A_ALLOCA]] : !cir.ptr<!s32i>, !s32i) map_clauses(to) capture(ByRef) -> !cir.ptr<!s32i> {name = "a"}
+  // CIR-HOST-NEXT: %[[MAP_B:.*]] = omp.map.info var_ptr(%[[B_ALLOCA]] : !cir.ptr<!s32i>, !s32i) map_clauses(from) capture(ByRef) -> !cir.ptr<!s32i> {name = "b"}
+  // CIR-HOST-NEXT: omp.target kernel_type(generic) map_entries(%[[MAP_A]] -> %[[ARG_A:.*]], %[[MAP_B]] -> %[[ARG_B:.*]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CIR-HOST: omp.terminator
+  // CIR-HOST-NEXT: }
+
+  // CIR-DEVICE: cir.func{{.*}}@target_map_multiple
+  // CIR-DEVICE-DAG: %[[A_ALLOCA:.*]] = cir.alloca "a" align(4) init : !cir.ptr<!s32i, target_address_space(5)>
+  // CIR-DEVICE-DAG: %[[B_ALLOCA:.*]] = cir.alloca "b" align(4) init : !cir.ptr<!s32i, target_address_space(5)>
+  // CIR-DEVICE: %[[CAST_A:.*]] = cir.cast address_space %[[A_ALLOCA]] : !cir.ptr<!s32i, target_address_space(5)> -> !cir.ptr<!s32i>
+  // CIR-DEVICE: %[[MAP_A:.*]] = omp.map.info var_ptr(%[[CAST_A]] : !cir.ptr<!s32i>, !s32i) map_clauses(to) capture(ByRef) -> !cir.ptr<!s32i> {name = "a"}
+  // CIR-DEVICE: %[[CAST_B:.*]] = cir.cast address_space %[[B_ALLOCA]] : !cir.ptr<!s32i, target_address_space(5)> -> !cir.ptr<!s32i>
+  // CIR-DEVICE: %[[MAP_B:.*]] = omp.map.info var_ptr(%[[CAST_B]] : !cir.ptr<!s32i>, !s32i) map_clauses(from) capture(ByRef) -> !cir.ptr<!s32i> {name = "b"}
+  // CIR-DEVICE: omp.target kernel_type(generic) map_entries(%[[MAP_A]] -> %[[ARG_A:.*]], %[[MAP_B]] -> %[[ARG_B:.*]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CIR-DEVICE: omp.terminator
+  // CIR-DEVICE-NEXT: }
+#pragma omp target map(to : a) map(from : b)
+  {
+    b = a;
+  }
+}
+
+// TODO: Test implicit mapping. Currently NYI
diff --git a/clang/test/CXX/expr/expr.const/p6-2a.cpp b/clang/test/CXX/expr/expr.const/p6-2a.cpp
index a937474d53b22..e97f35c891717 100644
--- a/clang/test/CXX/expr/expr.const/p6-2a.cpp
+++ b/clang/test/CXX/expr/expr.const/p6-2a.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -std=c++2a -verify %s
+// RUN: %clang_cc1 -std=c++2a -verify %s -fexperimental-new-constant-interpreter
 
 constexpr int non_class = 42;
 constexpr int arr_non_class[5] = {1, 2, 3};
diff --git a/clang/test/CodeGen/amdgpu-feature-builtins-invalid-use.cpp b/clang/test/CodeGen/amdgpu-feature-builtins-invalid-use.cpp
deleted file mode 100644
index 78f18d3a37b46..0000000000000
--- a/clang/test/CodeGen/amdgpu-feature-builtins-invalid-use.cpp
+++ /dev/null
@@ -1,48 +0,0 @@
-// RUN: not %clang_cc1 -triple amdgcn-amd-amdhsa -target-cpu gfx900 -emit-llvm %s -o - 2>&1 | FileCheck %s
-// RUN: not %clang_cc1 -triple spirv64-amd-amdhsa -emit-llvm %s -o - 2>&1 | FileCheck %s
-
-bool predicate(bool x);
-void pass_by_value(__amdgpu_feature_predicate_t x);
-
-void invalid_uses(int *p, int x, const __amdgpu_feature_predicate_t &lv,
-                  __amdgpu_feature_predicate_t &&rv) {
-    // CHECK: error: 'a' has type __amdgpu_feature_predicate_t, which is not constructible
-    __amdgpu_feature_predicate_t a;
-    // CHECK: error: 'b' has type __amdgpu_feature_predicate_t, which is not constructible
-    __amdgpu_feature_predicate_t b = __builtin_amdgcn_processor_is("gfx906");
-    // CHECK: error: 'c' has type __amdgpu_feature_predicate_t, which is not constructible
-    __amdgpu_feature_predicate_t c = lv;
-    // CHECK: error: 'd' has type __amdgpu_feature_predicate_t, which is not constructible
-    __amdgpu_feature_predicate_t d = rv;
-    // CHECK: error: '__builtin_amdgcn_processor_is("gfx906")' must be explicitly cast to 'bool'; however, please note that this is almost always an error and that it prevents the effective guarding of target dependent code, and thus should be avoided
-    bool invalid_use_in_init_0 = __builtin_amdgcn_processor_is("gfx906");
-    // CHECK: error: 'x' has type __amdgpu_feature_predicate_t, which is not constructible
-    pass_by_value(__builtin_amdgcn_processor_is("gfx906"));
-    // CHECK: error: '__builtin_amdgcn_is_invocable(__builtin_amdgcn_s_sleep_var)' must be explicitly cast to 'bool'; however, please note that this is almost always an error and that it prevents the effective guarding of target dependent code, and thus should be avoided
-    bool invalid_use_in_init_1 = __builtin_amdgcn_is_invocable(__builtin_amdgcn_s_sleep_var);
-    // CHECK: error: '__builtin_amdgcn_processor_is("gfx906")' must be explicitly cast to 'bool'; however, please note that this is almost always an error and that it prevents the effective guarding of target dependent code, and thus should be avoided
-    if (bool invalid_use_in_init_2 = __builtin_amdgcn_processor_is("gfx906")) return;
-    // CHECK: error: '__builtin_amdgcn_processor_is("gfx1200")' must be explicitly cast to 'bool'; however, please note that this is almost always an error and that it prevents the effective guarding of target dependent code, and thus should be avoided
-    if (predicate(__builtin_amdgcn_processor_is("gfx1200"))) __builtin_amdgcn_s_sleep_var(x);
-}
-
-void invalid_invocations(int x, const char* str) {
-    // CHECK: error: the argument to __builtin_amdgcn_processor_is must be a valid AMDGCN processor identifier; 'not_an_amdgcn_gfx_id' is not valid
-    // CHECK-DAG: note: valid AMDGCN processor identifiers are: {{.*}}gfx{{.*}}
-    if (__builtin_amdgcn_processor_is("not_an_amdgcn_gfx_id")) return;
-    // CHECK: error: the argument to __builtin_amdgcn_processor_is must be a string literal
-    if (__builtin_amdgcn_processor_is(str)) return;
-    // CHECK: error: the argument to __builtin_amdgcn_is_invocable must be either a target agnostic builtin or an AMDGCN target specific builtin; {{.*}}__builtin_amdgcn_s_sleep_var{{.*}} is not valid
-    if (__builtin_amdgcn_is_invocable("__builtin_amdgcn_s_sleep_var")) return;
-    // CHECK: error: the argument to __builtin_amdgcn_is_invocable must be either a target agnostic builtin or an AMDGCN target specific builtin; {{.*}}str{{.*}} is not valid
-    else if (__builtin_amdgcn_is_invocable(str)) return;
-    // CHECK: error: the argument to __builtin_amdgcn_is_invocable must be either a target agnostic builtin or an AMDGCN target specific builtin; {{.*}}x{{.*}} is not valid
-    else if (__builtin_amdgcn_is_invocable(x)) return;
-    // CHECK: error: use of undeclared identifier '__builtin_ia32_pause'
-    else if (__builtin_amdgcn_is_invocable(__builtin_ia32_pause)) return;
-}
-
-bool return_needs_cast() {
-    // CHECK: error: '__builtin_amdgcn_processor_is("gfx900")' must be explicitly cast to 'bool'; however, please note that this is almost always an error and that it prevents the effective guarding of target dependent code, and thus should be avoided
-    return __builtin_amdgcn_processor_is("gfx900");
-}
diff --git a/clang/test/CodeGenCXX/const-init-cxx2a.cpp b/clang/test/CodeGenCXX/const-init-cxx2a.cpp
index 3c83a9c94adec..765f5b8d46e6f 100644
--- a/clang/test/CodeGenCXX/const-init-cxx2a.cpp
+++ b/clang/test/CodeGenCXX/const-init-cxx2a.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -o - %s -std=c++2a | FileCheck %s --implicit-check-not=cxx_global_var_init --implicit-check-not=cxa_atexit
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -o - %s -std=c++2a                                         | FileCheck %s --implicit-check-not=cxx_global_var_init --implicit-check-not=cxa_atexit
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -o - %s -std=c++2a -fexperimental-new-constant-interpreter | FileCheck %s --implicit-check-not=cxx_global_var_init --implicit-check-not=cxa_atexit
 
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-pch -o %t.pch %s -std=c++2a
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -include-pch %t.pch -x c++ /dev/null -emit-llvm -o - -std=c++2a | FileCheck %s --implicit-check-not=cxx_global_var_init --implicit-check-not=cxa_atexit
diff --git a/clang/test/CodeGenCXX/non-const-init-cxx2a.cpp b/clang/test/CodeGenCXX/non-const-init-cxx2a.cpp
index 1ebb595897b31..48594632d4d06 100644
--- a/clang/test/CodeGenCXX/non-const-init-cxx2a.cpp
+++ b/clang/test/CodeGenCXX/non-const-init-cxx2a.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -triple x86_64-apple-darwin -emit-llvm -o - %s -std=c++2a | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-apple-darwin -emit-llvm -o - %s -std=c++2a -fexperimental-new-constant-interpreter | FileCheck %s
 
 // RUN: %clang_cc1 -triple x86_64-apple-darwin -emit-pch -o %t.pch %s -std=c++2a
 // RUN: %clang_cc1 -triple x86_64-apple-darwin -include-pch %t.pch -x c++ /dev/null -emit-llvm -o - -std=c++2a | FileCheck %s
diff --git a/clang/test/CodeGenOpenCL/address-spaces-conversions.cl b/clang/test/CodeGenOpenCL/address-spaces-conversions.cl
index f3a22fe89eaeb..af0c40e1417b2 100644
--- a/clang/test/CodeGenOpenCL/address-spaces-conversions.cl
+++ b/clang/test/CodeGenOpenCL/address-spaces-conversions.cl
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -O0 -ffake-address-space-map -cl-std=CL2.0 -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -O0 -ffake-address-space-map -cl-std=CL3.0 -cl-ext=+__opencl_c_generic_address_space -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -O0 -cl-std=CL2.0 -emit-llvm -o - | FileCheck --check-prefix=CHECK-NOFAKE %s
-// RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -O0 -cl-std=CL3.0 -cl-ext=+__opencl_c_generic_address_space -emit-llvm -o - | FileCheck --check-prefix=CHECK-NOFAKE %s
+// RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -O0 -ffake-address-space-map -cl-std=CL2.0 -emit-llvm -o - -Wno-deprecated-attributes | FileCheck %s
+// RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -O0 -ffake-address-space-map -cl-std=CL3.0 -cl-ext=+__opencl_c_generic_address_space -emit-llvm -o - -Wno-deprecated-attributes | FileCheck %s
+// RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -O0 -cl-std=CL2.0 -emit-llvm -o - -Wno-deprecated-attributes | FileCheck --check-prefix=CHECK-NOFAKE %s
+// RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -O0 -cl-std=CL3.0 -cl-ext=+__opencl_c_generic_address_space -emit-llvm -o - -Wno-deprecated-attributes | FileCheck --check-prefix=CHECK-NOFAKE %s
 // When -ffake-address-space-map is not used, all addr space mapped to 0 for x86_64.
 
 // test that we generate address space casts everywhere we need conversions of
diff --git a/clang/test/CodeGenOpenCL/address-spaces.cl b/clang/test/CodeGenOpenCL/address-spaces.cl
index b9f01069fa26c..072ce8f922937 100644
--- a/clang/test/CodeGenOpenCL/address-spaces.cl
+++ b/clang/test/CodeGenOpenCL/address-spaces.cl
@@ -1,15 +1,15 @@
-// RUN: %clang_cc1 %s -O0 -ffake-address-space-map -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK,SPIR
-// RUN: %clang_cc1 %s -O0 -cl-std=CL3.0 -cl-ext=-all -ffake-address-space-map -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK,SPIR
-// RUN: %clang_cc1 %s -O0 -cl-std=clc++2021 -cl-ext=-all -ffake-address-space-map -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK,SPIR
-// RUN: %clang_cc1 %s -O0 -DCL20 -cl-std=CL2.0 -ffake-address-space-map -emit-llvm -o - | FileCheck %s --check-prefixes=CL20,CL20SPIR
-// RUN: %clang_cc1 %s -O0 -triple amdgcn-amd-mesa3d -emit-llvm -o - | FileCheck --check-prefixes=CHECK,AMDGCN %s
-// RUN: %clang_cc1 %s -O0 -triple amdgcn-amd-mesa3d -cl-std=CL3.0 -emit-llvm -o - | FileCheck --check-prefixes=CHECK,AMDGCN %s
-// RUN: %clang_cc1 %s -O0 -triple amdgcn-amd-amdhsa -DCL20 -cl-std=CL2.0 -emit-llvm -o - | FileCheck %s --check-prefixes=CL20,CL20AMDGCN
-// RUN: %clang_cc1 %s -O0 -triple amdgcn-amd-amdhsa -DCL20 -cl-std=CL3.0 -emit-llvm -o - | FileCheck %s --check-prefixes=CL20,CL20AMDGCN
-// RUN: %clang_cc1 %s -O0 -triple amdgcn-mesa-mesa3d -emit-llvm -o - | FileCheck --check-prefixes=CHECK,AMDGCN %s
-// RUN: %clang_cc1 %s -O0 -triple amdgcn-mesa-mesa3d -cl-std=CL3.0 -emit-llvm -o - | FileCheck --check-prefixes=CHECK,AMDGCN %s
-// RUN: %clang_cc1 %s -O0 -triple r600-- -emit-llvm -o - | FileCheck --check-prefixes=CHECK,AMDGCN %s
-// RUN: %clang_cc1 %s -O0 -triple r600-- -emit-llvm -cl-std=CL3.0 -o - | FileCheck --check-prefixes=CHECK,AMDGCN %s
+// RUN: %clang_cc1 %s -O0 -ffake-address-space-map -emit-llvm -o - -Wno-deprecated-attributes | FileCheck %s --check-prefixes=CHECK,SPIR
+// RUN: %clang_cc1 %s -O0 -cl-std=CL3.0 -cl-ext=-all -ffake-address-space-map -emit-llvm -o - -Wno-deprecated-attributes | FileCheck %s --check-prefixes=CHECK,SPIR
+// RUN: %clang_cc1 %s -O0 -cl-std=clc++2021 -cl-ext=-all -ffake-address-space-map -emit-llvm -o - -Wno-deprecated-attributes | FileCheck %s --check-prefixes=CHECK,SPIR
+// RUN: %clang_cc1 %s -O0 -DCL20 -cl-std=CL2.0 -ffake-address-space-map -emit-llvm -o - -Wno-deprecated-attributes | FileCheck %s --check-prefixes=CL20,CL20SPIR
+// RUN: %clang_cc1 %s -O0 -triple amdgcn-amd-mesa3d -emit-llvm -o - -Wno-deprecated-attributes | FileCheck --check-prefixes=CHECK,AMDGCN %s
+// RUN: %clang_cc1 %s -O0 -triple amdgcn-amd-mesa3d -cl-std=CL3.0 -emit-llvm -o - -Wno-deprecated-attributes | FileCheck --check-prefixes=CHECK,AMDGCN %s
+// RUN: %clang_cc1 %s -O0 -triple amdgcn-amd-amdhsa -DCL20 -cl-std=CL2.0 -emit-llvm -o - -Wno-deprecated-attributes | FileCheck %s --check-prefixes=CL20,CL20AMDGCN
+// RUN: %clang_cc1 %s -O0 -triple amdgcn-amd-amdhsa -DCL20 -cl-std=CL3.0 -emit-llvm -o - -Wno-deprecated-attributes | FileCheck %s --check-prefixes=CL20,CL20AMDGCN
+// RUN: %clang_cc1 %s -O0 -triple amdgcn-mesa-mesa3d -emit-llvm -o - -Wno-deprecated-attributes | FileCheck --check-prefixes=CHECK,AMDGCN %s
+// RUN: %clang_cc1 %s -O0 -triple amdgcn-mesa-mesa3d -cl-std=CL3.0 -emit-llvm -o - -Wno-deprecated-attributes | FileCheck --check-prefixes=CHECK,AMDGCN %s
+// RUN: %clang_cc1 %s -O0 -triple r600-- -emit-llvm -o - -Wno-deprecated-attributes | FileCheck --check-prefixes=CHECK,AMDGCN %s
+// RUN: %clang_cc1 %s -O0 -triple r600-- -emit-llvm -cl-std=CL3.0 -o - -Wno-deprecated-attributes | FileCheck --check-prefixes=CHECK,AMDGCN %s
 
 // SPIR: %struct.S = type { i32, i32, ptr }
 // CL20SPIR: %struct.S = type { i32, i32, ptr addrspace(4) }
diff --git a/clang/test/CodeGenOpenCL/amdgpu-sizeof-alignof.cl b/clang/test/CodeGenOpenCL/amdgpu-sizeof-alignof.cl
index 3bd395da6d452..1761ae3ac984b 100644
--- a/clang/test/CodeGenOpenCL/amdgpu-sizeof-alignof.cl
+++ b/clang/test/CodeGenOpenCL/amdgpu-sizeof-alignof.cl
@@ -1,21 +1,17 @@
 // RUN: %clang_cc1 -triple r600 -cl-std=CL1.2 %s -emit-llvm -o - | FileCheck %s
 // RUN: %clang_cc1 -triple amdgcn-mesa-mesa3d -cl-std=CL1.2 %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -triple amdgcn---opencl -cl-std=CL1.2 %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -triple amdgcn---opencl -cl-std=CL2.0 %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -triple amdgcn---amdgizcl -cl-std=CL1.2 %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -triple amdgcn---amdgizcl -cl-std=CL2.0 %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -cl-std=CL1.2 %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -cl-std=CL2.0 %s -emit-llvm -o - | FileCheck %s
 
 // RUN: %clang_cc1 -triple r600 -cl-std=CL3.0 %s -emit-llvm -o - | FileCheck %s
 // RUN: %clang_cc1 -triple amdgcn-mesa-mesa3d -cl-std=CL3.0 %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -triple amdgcn---opencl -cl-std=CL3.0 %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -triple amdgcn---amdgizcl -cl-std=CL3.0 %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -cl-std=CL3.0 %s -emit-llvm -o - | FileCheck %s
 // RUN: %clang_cc1 -triple amdgcn-mesa-mesa3d -cl-ext=+__opencl_c_generic_address_space -cl-std=CL3.0 %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -triple amdgcn---opencl -cl-ext=+__opencl_c_generic_address_space -cl-std=CL3.0 %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -triple amdgcn---amdgizcl -cl-ext=+__opencl_c_generic_address_space -cl-std=CL3.0 %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -cl-ext=+__opencl_c_generic_address_space -cl-std=CL3.0 %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -triple amdgcn-amd-amdpal -cl-ext=+__opencl_c_generic_address_space -cl-std=CL3.0 %s -emit-llvm -o - | FileCheck %s
 // RUN: %clang_cc1 -triple r600 -cl-ext=+cl_khr_fp64,+__opencl_c_fp64 -cl-std=CL3.0 %s -emit-llvm -o - | FileCheck %s
 // RUN: %clang_cc1 -triple amdgcn-mesa-mesa3d -cl-ext=+cl_khr_fp64,+__opencl_c_fp64 -cl-std=CL3.0 %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -triple amdgcn---opencl -cl-ext=+cl_khr_fp64,+__opencl_c_fp64 -cl-std=CL3.0 %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -triple amdgcn---amdgizcl -cl-ext=+cl_khr_fp64,+__opencl_c_fp64 -cl-std=CL3.0 %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -cl-ext=+cl_khr_fp64,+__opencl_c_fp64 -cl-std=CL3.0 %s -emit-llvm -o - | FileCheck %s
 
 #ifdef __AMDGCN__
 #define PTSIZE 8
diff --git a/clang/test/CodeGenOpenCL/func-call-dbg-loc.cl b/clang/test/CodeGenOpenCL/func-call-dbg-loc.cl
index 6b56cbb35f3fd..60bf3916e9937 100644
--- a/clang/test/CodeGenOpenCL/func-call-dbg-loc.cl
+++ b/clang/test/CodeGenOpenCL/func-call-dbg-loc.cl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple amdgcn---amdgizcl -debug-info-kind=limited -gno-column-info -O0 -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -debug-info-kind=limited -gno-column-info -O0 -emit-llvm -o - %s | FileCheck %s
 
 typedef struct
 {
diff --git a/clang/test/CodeGenSYCL/address-space-conversions.cpp b/clang/test/CodeGenSYCL/address-space-conversions.cpp
index 506a24fb4a3ba..138a920880161 100644
--- a/clang/test/CodeGenSYCL/address-space-conversions.cpp
+++ b/clang/test/CodeGenSYCL/address-space-conversions.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple spir64 -fsycl-is-device -disable-llvm-passes -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple spir64 -fsycl-is-device -disable-llvm-passes -emit-llvm %s -o - -Wno-deprecated-attributes | FileCheck %s
 void bar(int &Data) {}
 // CHECK-DAG: define{{.*}} void @[[RAW_REF:[a-zA-Z0-9_]+]](ptr addrspace(4) noundef align 4 dereferenceable(4) %
 void bar2(int &Data) {}
diff --git a/clang/test/CodeGenSYCL/amd-address-space-conversions.cpp b/clang/test/CodeGenSYCL/amd-address-space-conversions.cpp
index 17a98195318ad..beb675e75e5a2 100644
--- a/clang/test/CodeGenSYCL/amd-address-space-conversions.cpp
+++ b/clang/test/CodeGenSYCL/amd-address-space-conversions.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -fsycl-is-device -disable-llvm-passes -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -fsycl-is-device -disable-llvm-passes -emit-llvm %s -o - -Wno-deprecated-attributes | FileCheck %s
 void bar(int &Data) {}
 // CHECK-DAG: define {{.*}} void @[[RAW_REF:[a-zA-Z0-9_]+]](ptr noundef nonnull align 4 dereferenceable(4) %
 void bar2(int &Data) {}
diff --git a/clang/test/CodeGenSYCL/cuda-address-space-conversions.cpp b/clang/test/CodeGenSYCL/cuda-address-space-conversions.cpp
index ffb601e62c118..ca3b7eca7cb4f 100644
--- a/clang/test/CodeGenSYCL/cuda-address-space-conversions.cpp
+++ b/clang/test/CodeGenSYCL/cuda-address-space-conversions.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -fsycl-is-device -disable-llvm-passes -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -fsycl-is-device -disable-llvm-passes -emit-llvm %s -o - -Wno-deprecated-attributes | FileCheck %s
 void bar(int &Data) {}
 // CHECK-DAG: define {{.*}} void @[[RAW_REF:[a-zA-Z0-9_]+]](ptr noundef nonnull align 4 dereferenceable(4) %
 void bar2(int &Data) {}
diff --git a/clang/test/Sema/LifetimeSafety/safety.cpp b/clang/test/Sema/LifetimeSafety/safety.cpp
index b59fac191dcfb..2c2d62b25b6c8 100644
--- a/clang/test/Sema/LifetimeSafety/safety.cpp
+++ b/clang/test/Sema/LifetimeSafety/safety.cpp
@@ -3955,3 +3955,9 @@ struct [[gsl::Pointer()]] PtrWithInt { int x; };
 PtrWithInt f() {
   return PtrWithInt{10};
 }
+
+// This would normally trigger a suggestion warning if -Wlifetime-safety-suggestions was on.
+// Since it is off, we expect NO warnings or notes here.
+View suggestion_disabled_test(View a) {
+  return a;
+}
diff --git a/clang/test/SemaCXX/amdgpu-feature-builtins-invalid-use.cpp b/clang/test/SemaCXX/amdgpu-feature-builtins-invalid-use.cpp
new file mode 100644
index 0000000000000..e4036ed839cab
--- /dev/null
+++ b/clang/test/SemaCXX/amdgpu-feature-builtins-invalid-use.cpp
@@ -0,0 +1,50 @@
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -target-cpu gfx900 -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -fsyntax-only -verify=expected,spirv %s
+
+bool predicate(bool x);
+void pass_by_value(__amdgpu_feature_predicate_t x);
+
+void invalid_uses(int *p, int x, const __amdgpu_feature_predicate_t &lv,
+                  __amdgpu_feature_predicate_t &&rv) {
+    __amdgpu_feature_predicate_t a;
+    // expected-error@-1 {{'a' has type __amdgpu_feature_predicate_t, which is not constructible}}
+    __amdgpu_feature_predicate_t b = __builtin_amdgcn_processor_is("gfx906");
+    // expected-error@-1 {{'b' has type __amdgpu_feature_predicate_t, which is not constructible}}
+    __amdgpu_feature_predicate_t c = lv;
+    // expected-error@-1 {{'c' has type __amdgpu_feature_predicate_t, which is not constructible}}
+    __amdgpu_feature_predicate_t d = rv;
+    // expected-error@-1 {{'d' has type __amdgpu_feature_predicate_t, which is not constructible}}
+    bool invalid_use_in_init_0 = __builtin_amdgcn_processor_is("gfx906");
+    // expected-error@-1 {{'__builtin_amdgcn_processor_is("gfx906")' must be explicitly cast to 'bool'; however, please note that this is almost always an error and that it prevents the effective guarding of target dependent code, and thus should be avoided}}
+    pass_by_value(__builtin_amdgcn_processor_is("gfx906"));
+    // expected-error@-1 {{'x' has type __amdgpu_feature_predicate_t, which is not constructible}}
+    bool invalid_use_in_init_1 = __builtin_amdgcn_is_invocable(__builtin_amdgcn_s_sleep_var);
+    // expected-error@-1 {{'__builtin_amdgcn_is_invocable(__builtin_amdgcn_s_sleep_var)' must be explicitly cast to 'bool'; however, please note that this is almost always an error and that it prevents the effective guarding of target dependent code, and thus should be avoided}}
+    if (bool invalid_use_in_init_2 = __builtin_amdgcn_processor_is("gfx906")) return;
+    // expected-error@-1 {{'__builtin_amdgcn_processor_is("gfx906")' must be explicitly cast to 'bool'; however, please note that this is almost always an error and that it prevents the effective guarding of target dependent code, and thus should be avoided}}
+    if (predicate(__builtin_amdgcn_processor_is("gfx1200"))) __builtin_amdgcn_s_sleep_var(x);
+    // expected-error@-1 {{no matching function for call to 'predicate'}}
+    // expected-error@-2 {{'__builtin_amdgcn_processor_is("gfx1200")' must be explicitly cast to 'bool'; however, please note that this is almost always an error and that it prevents the effective guarding of target dependent code, and thus should be avoided}}
+    // spirv-error@-3 {{'__builtin_amdgcn_s_sleep_var' cannot be invoked in the current context, as it requires the 'gfx12-insts' feature(s)}}
+}
+
+void invalid_invocations(int x, const char* str) {
+    if (__builtin_amdgcn_processor_is("not_an_amdgcn_gfx_id")) return;
+    // expected-error@-1 {{the argument to __builtin_amdgcn_processor_is must be a valid AMDGCN processor identifier; 'not_an_amdgcn_gfx_id' is not valid}}
+    // expected-note-re@-2 {{valid AMDGCN processor identifiers are: {{.*gfx.*}}}}
+    if (__builtin_amdgcn_processor_is(str)) return;
+    // expected-error@-1 {{the argument to __builtin_amdgcn_processor_is must be a string literal}}
+    if (__builtin_amdgcn_is_invocable("__builtin_amdgcn_s_sleep_var")) return;
+    // expected-error-re@-1 {{the argument to __builtin_amdgcn_is_invocable must be either a target agnostic builtin or an AMDGCN target specific builtin; {{.*__builtin_amdgcn_s_sleep_var.*}} is not valid}}
+    else if (__builtin_amdgcn_is_invocable(str)) return;
+    // expected-error-re@-1 {{the argument to __builtin_amdgcn_is_invocable must be either a target agnostic builtin or an AMDGCN target specific builtin; {{.*str.*}} is not valid}}
+    else if (__builtin_amdgcn_is_invocable(x)) return;
+    // expected-error-re@-1 {{the argument to __builtin_amdgcn_is_invocable must be either a target agnostic builtin or an AMDGCN target specific builtin; {{.*x.*}} is not valid}}
+    else if (__builtin_amdgcn_is_invocable(__builtin_ia32_pause)) return;
+    // expected-error@-1 {{use of undeclared identifier '__builtin_ia32_pause'}}
+}
+
+bool return_needs_cast() {
+    return __builtin_amdgcn_processor_is("gfx900");
+    // expected-error@-1 {{'__builtin_amdgcn_processor_is("gfx900")' must be explicitly cast to 'bool'; however, please note that this is almost always an error and that it prevents the effective guarding of target dependent code, and thus should be avoided}}
+}
diff --git a/clang/test/SemaCXX/constant-expression-p2280r4.cpp b/clang/test/SemaCXX/constant-expression-p2280r4.cpp
index bb5499e57ac78..77a9891e6620c 100644
--- a/clang/test/SemaCXX/constant-expression-p2280r4.cpp
+++ b/clang/test/SemaCXX/constant-expression-p2280r4.cpp
@@ -205,8 +205,8 @@ int f() {
 namespace uninit_reference_used {
   int y;
   constexpr int &r = r; // expected-error {{must be initialized by a constant expression}} \
-  // expected-note {{initializer of 'r' is not a constant expression}} \
-  // expected-note {{declared here}}
+                        // expected-note {{initializer of 'r' is not a constant expression}} \
+                        // expected-note {{declared here}}
   constexpr int &rr = (rr, y);
   constexpr int &g() {
     int &x = x; // expected-warning {{reference 'x' is not yet bound to a value when used within its own initialization}} \
@@ -224,12 +224,12 @@ namespace uninit_reference_used {
   // expected-note {{in call to 'g2()'}}
   constexpr int &g3() {
     int &x = (x,y); // expected-warning{{left operand of comma operator has no effect}} \
-    // expected-warning {{reference 'x' is not yet bound to a value when used within its own initialization}} \
-    // nointerpreter-note {{use of reference outside its lifetime is not allowed in a constant expression}}
+                    // expected-warning {{reference 'x' is not yet bound to a value when used within its own initialization}} \
+                    // expected-note {{use of reference outside its lifetime is not allowed in a constant expression}}
     return x;
   }
-  constexpr int &gg3 = g3(); // nointerpreter-error {{must be initialized by a constant expression}} \
-  // nointerpreter-note {{in call to 'g3()'}}
+  constexpr int &gg3 = g3(); // expected-error {{must be initialized by a constant expression}} \
+                             // expected-note {{in call to 'g3()'}}
   typedef decltype(sizeof(1)) uintptr_t;
   constexpr uintptr_t g4() {
     uintptr_t * &x = x; // expected-warning {{reference 'x' is not yet bound to a value when used within its own initialization}} \
diff --git a/clang/test/SemaOpenCL/usm-address-spaces-conversions.cl b/clang/test/SemaOpenCL/usm-address-spaces-conversions.cl
index 668b0db70d49e..8817e1ab42ee2 100644
--- a/clang/test/SemaOpenCL/usm-address-spaces-conversions.cl
+++ b/clang/test/SemaOpenCL/usm-address-spaces-conversions.cl
@@ -24,14 +24,14 @@
 #endif // CONSTANT
 
 void test(AS_COMP int *arg_comp,
-          __attribute__((opencl_global_device)) int *arg_device,
-          __attribute__((opencl_global_host)) int *arg_host) {
+          __attribute__((opencl_global_device)) int *arg_device, // expected-warning {{''opencl_global_device'' attribute is deprecated}}
+          __attribute__((opencl_global_host)) int *arg_host) { // expected-warning {{''opencl_global_host'' attribute is deprecated}}
   AS_COMP int *var_glob1 = arg_device;
   AS_COMP int *var_glob2 = arg_host;
   AS_COMP int *var_glob3 = (AS_COMP int *)arg_device;
   AS_COMP int *var_glob4 = (AS_COMP int *)arg_host;
-  arg_device = (__attribute__((opencl_global_device)) int *)arg_comp;
-  arg_host = (__attribute__((opencl_global_host)) int *)arg_comp;
+  arg_device = (__attribute__((opencl_global_device)) int *)arg_comp; // expected-warning {{''opencl_global_device'' attribute is deprecated}}
+  arg_host = (__attribute__((opencl_global_host)) int *)arg_comp; // expected-warning {{''opencl_global_host'' attribute is deprecated}}
 #ifdef GENERIC
   // expected-error@+6{{assigning '__generic int *__private' to '__global_device int *__private' changes address space of pointer}}
   // expected-error@+6{{assigning '__generic int *__private' to '__global_host int *__private' changes address space of pointer}}
@@ -61,7 +61,7 @@ void test(AS_COMP int *arg_comp,
   AS_INCOMP int *var_incomp1 = arg_device;
   AS_INCOMP int *var_incomp2 = arg_host;
   AS_INCOMP int *var_incomp3 =
-      (__attribute__((opencl_global_device)) int *)arg_device;
+      (__attribute__((opencl_global_device)) int *)arg_device; // expected-warning {{''opencl_global_device'' attribute is deprecated}}
   AS_INCOMP int *var_incomp4 =
-      (__attribute__((opencl_global_host)) int *)arg_host;
+      (__attribute__((opencl_global_host)) int *)arg_host; // expected-warning {{''opencl_global_host'' attribute is deprecated}}
 }
diff --git a/clang/test/SemaSYCL/address-space-conversions.cpp b/clang/test/SemaSYCL/address-space-conversions.cpp
index d8758248499de..e1a078338d523 100644
--- a/clang/test/SemaSYCL/address-space-conversions.cpp
+++ b/clang/test/SemaSYCL/address-space-conversions.cpp
@@ -65,16 +65,16 @@ void usages() {
   (void)i;
   (void)v;
 
-  __attribute__((opencl_global_host)) int *GLOB_HOST;
+  __attribute__((opencl_global_host)) int *GLOB_HOST; // expected-warning {{''opencl_global_host'' attribute is deprecated}}
   bar(*GLOB_HOST);
   bar2(*GLOB_HOST);
   GLOB = GLOB_HOST;
   GLOB_HOST = GLOB; // expected-error {{assigning '__global int *' to '__global_host int *' changes address space of pointer}}
-  GLOB_HOST = static_cast<__attribute__((opencl_global_host)) int *>(GLOB); // expected-error {{static_cast from '__global int *' to '__global_host int *' is not allowed}}
-  __attribute__((opencl_global_device)) int *GLOB_DEVICE;
+  GLOB_HOST = static_cast<__attribute__((opencl_global_host)) int *>(GLOB); // expected-warning {{''opencl_global_host'' attribute is deprecated}} expected-error {{static_cast from '__global int *' to '__global_host int *' is not allowed}}
+  __attribute__((opencl_global_device)) int *GLOB_DEVICE; // expected-warning {{''opencl_global_device'' attribute is deprecated}}
   bar(*GLOB_DEVICE);
   bar2(*GLOB_DEVICE);
   GLOB = GLOB_DEVICE;
   GLOB_DEVICE = GLOB; // expected-error {{assigning '__global int *' to '__global_device int *' changes address space of pointer}}
-  GLOB_DEVICE = static_cast<__attribute__((opencl_global_device)) int *>(GLOB); // expected-error {{static_cast from '__global int *' to '__global_device int *' is not allowed}}
+  GLOB_DEVICE = static_cast<__attribute__((opencl_global_device)) int *>(GLOB); // expected-warning {{''opencl_global_device'' attribute is deprecated}} expected-error {{static_cast from '__global int *' to '__global_device int *' is not allowed}}
 }
diff --git a/clang/unittests/AST/CommentLexer.cpp b/clang/unittests/AST/CommentLexer.cpp
index 99f469173964e..1d016cee2877b 100644
--- a/clang/unittests/AST/CommentLexer.cpp
+++ b/clang/unittests/AST/CommentLexer.cpp
@@ -447,22 +447,22 @@ TEST_F(CommentLexerTest, DoxygenCommand9) {
   ASSERT_EQ(tok::text,        Toks[0].getKind());
   ASSERT_EQ(StringRef(" "),   Toks[0].getText());
 
-  ASSERT_EQ(tok::unknown_command, Toks[1].getKind());
+  ASSERT_EQ(tok::unknown_backslash_command, Toks[1].getKind());
   ASSERT_EQ(StringRef("aaa"), Toks[1].getUnknownCommandName());
 
-  ASSERT_EQ(tok::unknown_command, Toks[2].getKind());
+  ASSERT_EQ(tok::unknown_backslash_command, Toks[2].getKind());
   ASSERT_EQ(StringRef("bbb"), Toks[2].getUnknownCommandName());
 
   ASSERT_EQ(tok::text,        Toks[3].getKind());
   ASSERT_EQ(StringRef(" "),   Toks[3].getText());
 
-  ASSERT_EQ(tok::unknown_command, Toks[4].getKind());
+  ASSERT_EQ(tok::unknown_backslash_command, Toks[4].getKind());
   ASSERT_EQ(StringRef("ccc"), Toks[4].getUnknownCommandName());
 
   ASSERT_EQ(tok::text,        Toks[5].getKind());
   ASSERT_EQ(StringRef("\t"),  Toks[5].getText());
 
-  ASSERT_EQ(tok::unknown_command, Toks[6].getKind());
+  ASSERT_EQ(tok::unknown_backslash_command, Toks[6].getKind());
   ASSERT_EQ(StringRef("ddd"), Toks[6].getUnknownCommandName());
 
   ASSERT_EQ(tok::newline,     Toks[7].getKind());
@@ -485,6 +485,38 @@ TEST_F(CommentLexerTest, DoxygenCommand10) {
   ASSERT_EQ(tok::newline,   Toks[2].getKind());
 }
 
+TEST_F(CommentLexerTest, DoxygenCommand11) {
+  const char *Source = "/// @aaa@bbb @ccc\t@ddd\n";
+  std::vector<Token> Toks;
+
+  lexString(Source, Toks);
+
+  ASSERT_EQ(8U, Toks.size());
+
+  ASSERT_EQ(tok::text, Toks[0].getKind());
+  ASSERT_EQ(StringRef(" "), Toks[0].getText());
+
+  ASSERT_EQ(tok::unknown_at_command, Toks[1].getKind());
+  ASSERT_EQ(StringRef("aaa"), Toks[1].getUnknownCommandName());
+
+  ASSERT_EQ(tok::unknown_at_command, Toks[2].getKind());
+  ASSERT_EQ(StringRef("bbb"), Toks[2].getUnknownCommandName());
+
+  ASSERT_EQ(tok::text, Toks[3].getKind());
+  ASSERT_EQ(StringRef(" "), Toks[3].getText());
+
+  ASSERT_EQ(tok::unknown_at_command, Toks[4].getKind());
+  ASSERT_EQ(StringRef("ccc"), Toks[4].getUnknownCommandName());
+
+  ASSERT_EQ(tok::text, Toks[5].getKind());
+  ASSERT_EQ(StringRef("\t"), Toks[5].getText());
+
+  ASSERT_EQ(tok::unknown_at_command, Toks[6].getKind());
+  ASSERT_EQ(StringRef("ddd"), Toks[6].getUnknownCommandName());
+
+  ASSERT_EQ(tok::newline, Toks[7].getKind());
+}
+
 TEST_F(CommentLexerTest, RegisterCustomBlockCommand) {
   const char *Source =
     "/// \\NewBlockCommand Aaa.\n"
diff --git a/compiler-rt/lib/tysan/tysan.cpp b/compiler-rt/lib/tysan/tysan.cpp
index 0be4fe91a7938..8b3b60dc4cf83 100644
--- a/compiler-rt/lib/tysan/tysan.cpp
+++ b/compiler-rt/lib/tysan/tysan.cpp
@@ -331,8 +331,6 @@ __tysan_instrument_mem_inst(char *dest, char *src, uint64_t size,
 
   if (needsMemMove) {
     internal_memmove((char *)destShadowDataPtr, srcShadow, size << PtrShift());
-  } else {
-    internal_memcpy((char *)destShadowDataPtr, srcShadow, size << PtrShift());
   }
 }
 
diff --git a/compiler-rt/lib/tysan/tysan_interceptors.cpp b/compiler-rt/lib/tysan/tysan_interceptors.cpp
index 095ce0e74bb8b..1ba613d603ee8 100644
--- a/compiler-rt/lib/tysan/tysan_interceptors.cpp
+++ b/compiler-rt/lib/tysan/tysan_interceptors.cpp
@@ -53,18 +53,6 @@ INTERCEPTOR(void *, memmove, void *dst, const void *src, uptr size) {
   return res;
 }
 
-INTERCEPTOR(void *, memcpy, void *dst, const void *src, uptr size) {
-  if (!tysan_inited && REAL(memcpy) == nullptr) {
-    // memmove is used here because on some platforms this will also
-    // intercept the memmove implementation.
-    return internal_memmove(dst, src, size);
-  }
-
-  void *res = REAL(memcpy)(dst, src, size);
-  tysan_copy_types(dst, src, size);
-  return res;
-}
-
 INTERCEPTOR(void *, mmap, void *addr, SIZE_T length, int prot, int flags,
             int fd, OFF_T offset) {
   void *res = REAL(mmap)(addr, length, prot, flags, fd, offset);
@@ -238,7 +226,6 @@ void InitializeInterceptors() {
 
   INTERCEPT_FUNCTION(memset);
   INTERCEPT_FUNCTION(memmove);
-  INTERCEPT_FUNCTION(memcpy);
 
   inited = 1;
 }
diff --git a/compiler-rt/test/tysan/global.c b/compiler-rt/test/tysan/global.c
index 247ee768a8162..9fac976776410 100644
--- a/compiler-rt/test/tysan/global.c
+++ b/compiler-rt/test/tysan/global.c
@@ -12,20 +12,12 @@ int main() {
   // CHECK: WRITE of size 4 at {{.*}} with type int accesses an existing object of type float
   // CHECK: {{#0 0x.* in main .*global.c:}}[[@LINE-3]]
 
-  void *mem = malloc(sizeof(long));
-  *(int *)mem = 6;
-  memcpy(mem, &L, sizeof(L));
-  *(int *)mem = 8;
+  P = *(((float *)&L) + 1);
   // CHECK: ERROR: TypeSanitizer: type-aliasing-violation
-  // CHECK: WRITE of size 4 at {{.*}} with type int accesses an existing object of type long
+  // CHECK: READ of size 4 at {{.*}} with type float accesses part of an existing object of type long that starts at offset -4
   // CHECK: {{#0 0x.* in main .*global.c:}}[[@LINE-3]]
-  int r = *(((int *)mem) + 1);
-  // CHECK: ERROR: TypeSanitizer: type-aliasing-violation
-  // CHECK: READ of size 4 at {{.*}} with type int accesses part of an existing object of type long that starts at offset -4
-  // CHECK: {{#0 0x.* in main .*global.c:}}[[@LINE-3]]
-  free(mem);
 
-  return r;
+  return 0;
 }
 
 // CHECK-NOT: ERROR: TypeSanitizer: type-aliasing-violation
diff --git a/compiler-rt/test/tysan/memcpy_doesnt_change_shadow.c b/compiler-rt/test/tysan/memcpy_doesnt_change_shadow.c
new file mode 100644
index 0000000000000..2545acfa1fbb3
--- /dev/null
+++ b/compiler-rt/test/tysan/memcpy_doesnt_change_shadow.c
@@ -0,0 +1,25 @@
+// RUN: %clang_tysan -O0 %s -o %t && %run %t >%t.out 2>&1
+// RUN: FileCheck %s < %t.out
+
+#include <stdio.h>
+#include <string.h>
+
+int main() {
+
+  int i = 0;
+  float f = 5.0f;
+
+  *(float *)&i = 3.0f;
+  *(int *)&f = 2;
+  // CHECK: WRITE of size 4 at 0x{{.*}} with type float accesses an existing object of type int
+  // CHECK: WRITE of size 4 at 0x{{.*}} with type int accesses an existing object of type float
+
+  memcpy(&i, &f, sizeof(int));
+
+  *(float *)&i = 3.0f;
+  *(int *)&f = 2;
+  // CHECK: WRITE of size 4 at 0x{{.*}} with type float accesses an existing object of type int
+  // CHECK: WRITE of size 4 at 0x{{.*}} with type int accesses an existing object of type float
+
+  return 0;
+}
diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 1cb65c5c6f696..96dd0b1abe305 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -12,16 +12,10 @@ find_package(Backtrace)
 set(HAVE_BACKTRACE ${Backtrace_FOUND})
 set(BACKTRACE_HEADER ${Backtrace_HEADER})
 
-# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
-
-# Module sources that are required by other modules
-set(intrinsics_sources
-  __fortran_builtins.f90
-  __cuda_builtins.f90
-)
-
 # Fortran sources for builtin .mod files
 set(module_sources
+  __fortran_builtins.f90
+  __cuda_builtins.f90
   __fortran_ieee_exceptions.f90
   __fortran_type_info.f90
   f90deviceio.f90
@@ -218,10 +212,8 @@ file(GLOB_RECURSE private_headers
   )
 
 if (LLVM_TARGET_TRIPLE MATCHES "^ppc|^powerpc")
-  list(APPEND intrinsics_sources
-    __ppc_types.f90
-  )
   list(APPEND module_sources
+    __ppc_types.f90
     __ppc_intrinsics.f90
     mma.f90
   )
@@ -267,21 +259,6 @@ if (TARGET FortranFloat128MathILib)
 endif ()
 
 
-# When a target depends on an object library, CMake seems to try to only build
-# the object files that the target actual needs. If we are only interested
-# in the module files, nothing gets built at all. To ensure that the module
-# files are built, insert a custom target that is opaque to CMake so it cannot
-# apply this optimization. Dependees on module files must depend on this
-# barrier instead. An actual COMMAND (that does nothing) seems to be necessary
-# on Windows to work.
-function (add_module_barrier barriername objlib)
-  add_custom_target(${barriername}
-    COMMAND ${CMAKE_COMMAND} -E true
-  )
-  add_dependencies(${barriername} ${objlib})
-endfunction ()
-
-
 # Build module files if requested.
 # The object files written by Flang for these are unused. In the future parts
 # of flang-rt may itself be implemented in Fortran in which case these Fortran
@@ -289,50 +266,10 @@ endfunction ()
 # libflang_rt.runtime{.a/.so}. If they also provide an importable .mod, add them
 # to flang_module_target(... PUBLIC).
 if (FLANG_RT_FORTRAN_MODULES)
-  # CMake ignores intrinsic USE dependencies
-  # CMake has an option Fortran_BUILDING_INSTRINSIC_MODULES/Fortran_BUILDING_INTRINSIC_MODULES
-  # to disable this behavior, unfortunately it does not work with Ninja
-  # (https://gitlab.kitware.com/cmake/cmake/-/issues/26803)
-  # As a workaround, we build those intrinsic modules first such that the main
-  # runtime can depend on it. To ensure that modules files are also transitively
-  # updated if a USE'd .mod file changes (a .mod stores the checksums of all
-  # .mod files it depends on and therefore needs to be updated as well), inject
-  # an file-level dependency via OBJECT_DEPENDS.
-
-  add_flangrt_library(flang_rt.mod.fortran.builtins OBJECT
-    __fortran_builtins.f90
-  )
-  set_property(SOURCE __fortran_builtins.f90 PROPERTY OBJECT_OUTPUTS "${CMAKE_BINARY_DIR}/modules/${CMAKE_CFG_INTDIR}/__fortran_builtins${CMAKE_Fortran_OUTPUT_EXTENSION}")
-  flang_module_target(flang_rt.mod.fortran.builtins PUBLIC)
-  add_module_barrier(flang_rt.mod.fortran.builtins.barrier flang_rt.mod.fortran.builtins)
-
-  add_flangrt_library(flang_rt.mod.cuda.builtins OBJECT
-    __cuda_builtins.f90
-  )
-  set_property(SOURCE __cuda_builtins.f90 PROPERTY OBJECT_OUTPUTS "${CMAKE_BINARY_DIR}/modules/${CMAKE_CFG_INTDIR}/__cuda_builtins${CMAKE_Fortran_OUTPUT_EXTENSION}")
-  add_dependencies(flang_rt.mod.cuda.builtins flang_rt.mod.fortran.builtins.barrier)
-  set_property(SOURCE __cuda_builtins.f90
-    APPEND PROPERTY OBJECT_DEPENDS
-      "${CMAKE_BINARY_DIR}/modules/${CMAKE_CFG_INTDIR}/__fortran_builtins${CMAKE_Fortran_OUTPUT_EXTENSION}"
-  )
-  flang_module_target(flang_rt.mod.cuda.builtins PUBLIC)
-  add_module_barrier(flang_rt.mod.cuda.builtins.barrier flang_rt.mod.cuda.builtins)
-
-  # The modules themselves
-  add_flangrt_library(flang_rt.mod OBJECT
+  add_flangrt_library(flang-rt-mod OBJECT
     ${module_sources}
   )
-  add_dependencies(flang_rt.mod flang_rt.mod.fortran.builtins.barrier flang_rt.mod.cuda.builtins.barrier)
-  foreach(_srcfile IN LISTS module_sources)
-    set_property(SOURCE ${_srcfile}
-      APPEND PROPERTY OBJECT_DEPENDS
-        "${CMAKE_BINARY_DIR}/modules/${CMAKE_CFG_INTDIR}/__fortran_builtins${CMAKE_Fortran_OUTPUT_EXTENSION}"
-        "${CMAKE_BINARY_DIR}/modules/${CMAKE_CFG_INTDIR}/__cuda_builtins${CMAKE_Fortran_OUTPUT_EXTENSION}"
-    )
-  endforeach()
-
-  flang_module_target(flang_rt.mod PUBLIC)
-  add_module_barrier(flang-rt-mod flang_rt.mod)
+  flang_module_target(flang-rt-mod PUBLIC)
 
   if (RUNTIMES_FORTRAN_MODULES)
     set(_mod_install_dest "${RUNTIMES_INSTALL_RESOURCE_MOD_PATH}/..")
diff --git a/flang-rt/lib/runtime/__fortran_type_info.f90 b/flang-rt/lib/runtime/__fortran_type_info.f90
index 0f1c3d018d009..d8cf385b8665e 100644
--- a/flang-rt/lib/runtime/__fortran_type_info.f90
+++ b/flang-rt/lib/runtime/__fortran_type_info.f90
@@ -13,7 +13,7 @@
 
 module __fortran_type_info
 
-  use, intrinsic :: __fortran_builtins, &
+  use __fortran_builtins, &
     only: __builtin_c_ptr, __builtin_c_devptr, __builtin_c_funptr
   implicit none
 
diff --git a/flang-rt/lib/runtime/cooperative_groups.f90 b/flang-rt/lib/runtime/cooperative_groups.f90
index 5ca0c3aa1f3a5..5c141d4d254db 100644
--- a/flang-rt/lib/runtime/cooperative_groups.f90
+++ b/flang-rt/lib/runtime/cooperative_groups.f90
@@ -10,8 +10,8 @@
 
 module cooperative_groups
 
-use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
-use :: cudadevice ! implicit dependency, made explicit for CMake
+use __fortran_builtins, only: c_devptr => __builtin_c_devptr
+use cudadevice ! implicit dependency, made explicit for CMake
 
 implicit none
 
diff --git a/flang-rt/lib/runtime/cudadevice.f90 b/flang-rt/lib/runtime/cudadevice.f90
index be041ba9d0a71..ea4787dd62e40 100644
--- a/flang-rt/lib/runtime/cudadevice.f90
+++ b/flang-rt/lib/runtime/cudadevice.f90
@@ -10,9 +10,9 @@
 
 module cudadevice
   use __cuda_device
-  use, intrinsic :: __fortran_builtins, only: dim3 => __builtin_dim3
-  use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
-  use, intrinsic :: __fortran_builtins, only: c_devloc => __builtin_c_devloc
+  use __fortran_builtins, only: dim3 => __builtin_dim3
+  use __fortran_builtins, only: c_devptr => __builtin_c_devptr
+  use __fortran_builtins, only: c_devloc => __builtin_c_devloc
 implicit none
 
   ! Synchronization Functions
diff --git a/flang/lib/Optimizer/OpenACC/Support/FIROpenACCTypeInterfaces.cpp b/flang/lib/Optimizer/OpenACC/Support/FIROpenACCTypeInterfaces.cpp
index da9114200a5f2..e0ab6b148f11b 100644
--- a/flang/lib/Optimizer/OpenACC/Support/FIROpenACCTypeInterfaces.cpp
+++ b/flang/lib/Optimizer/OpenACC/Support/FIROpenACCTypeInterfaces.cpp
@@ -345,7 +345,9 @@ generateSeqTyAccBounds(fir::SequenceType seqType, mlir::Value var,
           firBuilder, loc, exv, info);
     }
 
-    assert(false && "array with unknown dimension expected to have descriptor");
+    // An assumed-size array (or other non-descriptor array with an unknown
+    // trailing extent) has no recoverable bounds here and is passed without a
+    // descriptor; map it without bounds rather than asserting.
     return {};
   }
 
diff --git a/flang/lib/Semantics/openmp-utils.cpp b/flang/lib/Semantics/openmp-utils.cpp
index cfdf2f8eb56c9..ffda807c2dc32 100644
--- a/flang/lib/Semantics/openmp-utils.cpp
+++ b/flang/lib/Semantics/openmp-utils.cpp
@@ -27,6 +27,7 @@
 #include "flang/Parser/openmp-utils.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/expression.h"
+#include "flang/Semantics/openmp-directive-sets.h"
 #include "flang/Semantics/scope.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Semantics/symbol.h"
@@ -2318,6 +2319,31 @@ UnsupportedSelectorFeature FindUnsupportedSelectorFeature(
   return UnsupportedSelectorFeature::None;
 }
 
+// Add the construct trait properties implied by an OpenMP directive (e.g.
+// `target` adds `construct_target_target`, `target teams` adds both
+// `construct_target_target` and `construct_teams_teams`) to \p vmi. This
+// decomposes combined/composite construct selectors into their leaf traits.
+static void AppendConstructTraitsForDirective(
+    llvm::omp::Directive dir, llvm::omp::VariantMatchInfo &vmi) {
+  auto add = [&](llvm::omp::TraitProperty prop) {
+    vmi.addTrait(prop, llvm::omp::getOpenMPContextTraitPropertyName(prop, ""));
+  };
+  if (llvm::omp::allTargetSet.test(dir))
+    add(llvm::omp::TraitProperty::construct_target_target);
+  if (llvm::omp::allTeamsSet.test(dir))
+    add(llvm::omp::TraitProperty::construct_teams_teams);
+  if (llvm::omp::allParallelSet.test(dir))
+    add(llvm::omp::TraitProperty::construct_parallel_parallel);
+  if (llvm::omp::allDoSet.test(dir))
+    add(llvm::omp::TraitProperty::construct_for_for);
+  if (llvm::omp::allSimdSet.test(dir))
+    add(llvm::omp::TraitProperty::construct_simd_simd);
+  // dispatch is a standalone construct trait (not part of any combined
+  // directive set), so it is matched explicitly.
+  if (dir == llvm::omp::Directive::OMPD_dispatch)
+    add(llvm::omp::TraitProperty::construct_dispatch_dispatch);
+}
+
 static void AddTraitPropertiesFromSelector(llvm::omp::TraitSet set,
     const parser::OmpTraitSelector &selector, llvm::omp::VariantMatchInfo &vmi,
     SemanticsContext &semaCtx,
@@ -2368,10 +2394,8 @@ static void AddTraitPropertiesFromSelector(llvm::omp::TraitSet set,
 
   // Construct trait selector with no properties (e.g. `construct={simd}`):
   // the selector itself implies the property.
-  llvm::omp::TraitProperty propKind{
-      llvm::omp::getOpenMPContextTraitPropertyForSelector(selectorKind)};
-  if (propKind != llvm::omp::TraitProperty::invalid) {
-    vmi.addTrait(set, propKind, traitName.ToString(), scorePtr);
+  if (const auto *dir{std::get_if<llvm::omp::Directive>(&traitName.u)}) {
+    AppendConstructTraitsForDirective(*dir, vmi);
   }
 }
 
diff --git a/flang/test/Fir/OpenACC/openacc-mappable.fir b/flang/test/Fir/OpenACC/openacc-mappable.fir
index 91a83915a0900..900ba2886c67e 100644
--- a/flang/test/Fir/OpenACC/openacc-mappable.fir
+++ b/flang/test/Fir/OpenACC/openacc-mappable.fir
@@ -141,4 +141,19 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<f16 = dense<16> : vector<2xi64>,
   // CHECK: fir.result %[[ZERO]], %[[MINUSONE]], %[[ZERO]], %[[ZERO]], %[[ZERO]] : index, index, index, index, index
   // CHECK: }
   // CHECK: Upper bound: %[[IF]]:5 = fir.if %[[PRESENT]] -> (index, index, index, index, index) {
+
+  // An assumed-size array (unknown trailing extent) passed without a
+  // descriptor, whose acc clause operand is not produced by an [hl]fir.declare
+  // (so no shape is recoverable). It must produce no bounds.
+  func.func @_QPassumed_size_no_descriptor(%arg0: !fir.ref<!fir.array<?x?xf32>> {fir.bindc_name = "a"}) {
+    %0 = fir.convert %arg0 : (!fir.ref<!fir.array<?x?xf32>>) -> !fir.ref<!fir.array<?x?xf32>>
+    %1 = acc.copyin varPtr(%0 : !fir.ref<!fir.array<?x?xf32>>) -> !fir.ref<!fir.array<?x?xf32>> {name = "a", structured = false}
+    acc.enter_data dataOperands(%1 : !fir.ref<!fir.array<?x?xf32>>)
+    return
+  }
+  // CHECK: Visiting: %{{.*}} = acc.copyin varPtr(%{{.*}} : !fir.ref<!fir.array<?x?xf32>>) -> !fir.ref<!fir.array<?x?xf32>> {name = "a", structured = false}
+  // CHECK: Pointer-like and Mappable: !fir.ref<!fir.array<?x?xf32>>
+  // CHECK: Type category: array
+  // CHECK: Has unknown dimensions: true
+  // CHECK-NOT: Bound[
 }
diff --git a/flang/test/Lower/OpenMP/metadirective-construct.f90 b/flang/test/Lower/OpenMP/metadirective-construct.f90
index 5c2fdf93b6057..d21585699cd5a 100644
--- a/flang/test/Lower/OpenMP/metadirective-construct.f90
+++ b/flang/test/Lower/OpenMP/metadirective-construct.f90
@@ -95,3 +95,22 @@ subroutine test_begin_construct_selected_parent()
     !$omp end metadirective
   !$omp end target
 end subroutine
+
+! CHECK-LABEL: func.func @_QPtest_construct_combined_directive(
+! CHECK:         omp.target
+! CHECK:           omp.parallel
+! CHECK-NOT:         omp.taskyield
+! CHECK:             omp.barrier
+! CHECK:             omp.terminator
+! CHECK:           omp.terminator
+! CHECK:         return
+subroutine test_construct_combined_directive()
+  !$omp target
+    !$omp parallel
+      !$omp metadirective &
+      !$omp & when(construct={parallel}: taskyield) &
+      !$omp & when(construct={target parallel}: barrier) &
+      !$omp & default(nothing)
+    !$omp end parallel
+  !$omp end target
+end subroutine
diff --git a/libc/config/baremetal/aarch64/entrypoints.txt b/libc/config/baremetal/aarch64/entrypoints.txt
index f64eb55b40acd..73f4c1f937388 100644
--- a/libc/config/baremetal/aarch64/entrypoints.txt
+++ b/libc/config/baremetal/aarch64/entrypoints.txt
@@ -865,6 +865,7 @@ list(APPEND TARGET_LIBM_ENTRYPOINTS
   libc.src.math.setpayloadbf16
   libc.src.math.setpayloadsigbf16
   libc.src.math.sqrtbf16
+  libc.src.math.tanbf16
   libc.src.math.truncbf16
   libc.src.math.totalorderbf16
   libc.src.math.totalordermagbf16
diff --git a/libc/config/baremetal/arm/entrypoints.txt b/libc/config/baremetal/arm/entrypoints.txt
index 55a175923faa1..a5c1b20a7a434 100644
--- a/libc/config/baremetal/arm/entrypoints.txt
+++ b/libc/config/baremetal/arm/entrypoints.txt
@@ -876,6 +876,7 @@ list(APPEND TARGET_LIBM_ENTRYPOINTS
   libc.src.math.setpayloadbf16
   libc.src.math.setpayloadsigbf16
   libc.src.math.sqrtbf16
+  libc.src.math.tanbf16
   libc.src.math.truncbf16
   libc.src.math.totalorderbf16
   libc.src.math.totalordermagbf16
diff --git a/libc/config/baremetal/riscv/entrypoints.txt b/libc/config/baremetal/riscv/entrypoints.txt
index 201d2e9cbf97b..60bc924993fc8 100644
--- a/libc/config/baremetal/riscv/entrypoints.txt
+++ b/libc/config/baremetal/riscv/entrypoints.txt
@@ -873,6 +873,7 @@ list(APPEND TARGET_LIBM_ENTRYPOINTS
   libc.src.math.setpayloadbf16
   libc.src.math.setpayloadsigbf16
   libc.src.math.sqrtbf16
+  libc.src.math.tanbf16
   libc.src.math.truncbf16
   libc.src.math.totalorderbf16
   libc.src.math.totalordermagbf16
diff --git a/libc/config/darwin/aarch64/entrypoints.txt b/libc/config/darwin/aarch64/entrypoints.txt
index 413534a14f34e..6f4916b203de5 100644
--- a/libc/config/darwin/aarch64/entrypoints.txt
+++ b/libc/config/darwin/aarch64/entrypoints.txt
@@ -686,6 +686,7 @@ list(APPEND TARGET_LIBM_ENTRYPOINTS
   libc.src.math.setpayloadbf16
   libc.src.math.setpayloadsigbf16
   libc.src.math.sqrtbf16
+  libc.src.math.tanbf16
   libc.src.math.truncbf16
   libc.src.math.totalorderbf16
   libc.src.math.totalordermagbf16
diff --git a/libc/config/darwin/x86_64/entrypoints.txt b/libc/config/darwin/x86_64/entrypoints.txt
index d35700d341d1a..c09ccadbb8c4c 100644
--- a/libc/config/darwin/x86_64/entrypoints.txt
+++ b/libc/config/darwin/x86_64/entrypoints.txt
@@ -304,6 +304,7 @@ list(APPEND TARGET_LIBM_ENTRYPOINTS
   libc.src.math.setpayloadbf16
   libc.src.math.setpayloadsigbf16
   libc.src.math.sqrtbf16
+  libc.src.math.tanbf16
   libc.src.math.truncbf16
   libc.src.math.totalorderbf16
   libc.src.math.totalordermagbf16
diff --git a/libc/config/freebsd/x86_64/entrypoints.txt b/libc/config/freebsd/x86_64/entrypoints.txt
index 394b83130f067..7b8b8384e6dd8 100644
--- a/libc/config/freebsd/x86_64/entrypoints.txt
+++ b/libc/config/freebsd/x86_64/entrypoints.txt
@@ -627,6 +627,7 @@ list(APPEND TARGET_LIBM_ENTRYPOINTS
   libc.src.math.setpayloadbf16
   libc.src.math.setpayloadsigbf16
   libc.src.math.sqrtbf16
+  libc.src.math.tanbf16
   libc.src.math.truncbf16
   libc.src.math.totalorderbf16
   libc.src.math.totalordermagbf16
diff --git a/libc/config/gpu/amdgpu/entrypoints.txt b/libc/config/gpu/amdgpu/entrypoints.txt
index 7fcab53bd99f3..60a98cf5c773f 100644
--- a/libc/config/gpu/amdgpu/entrypoints.txt
+++ b/libc/config/gpu/amdgpu/entrypoints.txt
@@ -711,6 +711,7 @@ list(APPEND TARGET_LIBM_ENTRYPOINTS
   libc.src.math.setpayloadbf16
   libc.src.math.setpayloadsigbf16
   libc.src.math.sqrtbf16
+  libc.src.math.tanbf16
   libc.src.math.truncbf16
   libc.src.math.totalorderbf16
   libc.src.math.totalordermagbf16
diff --git a/libc/config/gpu/nvptx/entrypoints.txt b/libc/config/gpu/nvptx/entrypoints.txt
index be1467e9cbceb..70bdbde97e868 100644
--- a/libc/config/gpu/nvptx/entrypoints.txt
+++ b/libc/config/gpu/nvptx/entrypoints.txt
@@ -713,6 +713,7 @@ list(APPEND TARGET_LIBM_ENTRYPOINTS
   libc.src.math.setpayloadbf16
   libc.src.math.setpayloadsigbf16
   libc.src.math.sqrtbf16
+  libc.src.math.tanbf16
   libc.src.math.truncbf16
   libc.src.math.totalorderbf16
   libc.src.math.totalordermagbf16
diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index 599ae32a35073..570f7666ba67b 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -986,6 +986,7 @@ list(APPEND TARGET_LIBM_ENTRYPOINTS
   libc.src.math.setpayloadbf16
   libc.src.math.setpayloadsigbf16
   libc.src.math.sqrtbf16
+  libc.src.math.tanbf16
   libc.src.math.truncbf16
   libc.src.math.totalorderbf16
   libc.src.math.totalordermagbf16
@@ -1177,6 +1178,7 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.stdlib.exit
     libc.src.stdlib.getenv
     libc.src.stdlib.setenv
+    libc.src.stdlib.unsetenv
     libc.src.stdlib.quick_exit
 
     # signal.h entrypoints
diff --git a/libc/config/linux/aarch64/headers.txt b/libc/config/linux/aarch64/headers.txt
index a7ce42c9c4a91..0f6687dcd8a2d 100644
--- a/libc/config/linux/aarch64/headers.txt
+++ b/libc/config/linux/aarch64/headers.txt
@@ -15,6 +15,7 @@ set(TARGET_PUBLIC_HEADERS
     libc.include.fenv
     libc.include.float
     libc.include.inttypes
+    libc.include.libgen
     libc.include.limits
     libc.include.link
     libc.include.locale
diff --git a/libc/config/linux/arm/entrypoints.txt b/libc/config/linux/arm/entrypoints.txt
index d610e663229ba..e6aa6091ed550 100644
--- a/libc/config/linux/arm/entrypoints.txt
+++ b/libc/config/linux/arm/entrypoints.txt
@@ -557,6 +557,7 @@ list(APPEND TARGET_LIBM_ENTRYPOINTS
   libc.src.math.setpayloadbf16
   libc.src.math.setpayloadsigbf16
   libc.src.math.sqrtbf16
+  libc.src.math.tanbf16
   libc.src.math.truncbf16
   libc.src.math.totalorderbf16
   libc.src.math.totalordermagbf16
diff --git a/libc/config/linux/arm/headers.txt b/libc/config/linux/arm/headers.txt
index c8ca848b79f09..50627958eb38c 100644
--- a/libc/config/linux/arm/headers.txt
+++ b/libc/config/linux/arm/headers.txt
@@ -6,6 +6,7 @@ set(TARGET_PUBLIC_HEADERS
     libc.include.fenv
     libc.include.float
     libc.include.inttypes
+    libc.include.libgen
     libc.include.malloc
     libc.include.math
     libc.include.search
diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index ef2d13e6d6f9e..2508de5cfe5a4 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -1007,6 +1007,7 @@ list(APPEND TARGET_LIBM_ENTRYPOINTS
   libc.src.math.setpayloadbf16
   libc.src.math.setpayloadsigbf16
   libc.src.math.sqrtbf16
+  libc.src.math.tanbf16
   libc.src.math.truncbf16
   libc.src.math.totalorderbf16
   libc.src.math.totalordermagbf16
@@ -1306,6 +1307,7 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.stdlib.exit
     libc.src.stdlib.getenv
     libc.src.stdlib.setenv
+    libc.src.stdlib.unsetenv
     libc.src.stdlib.quick_exit
 
     # signal.h entrypoints
diff --git a/libc/config/linux/riscv/headers.txt b/libc/config/linux/riscv/headers.txt
index ce9cda307caf3..03bc2abd9ddea 100644
--- a/libc/config/linux/riscv/headers.txt
+++ b/libc/config/linux/riscv/headers.txt
@@ -15,6 +15,7 @@ set(TARGET_PUBLIC_HEADERS
     libc.include.fenv
     libc.include.float
     libc.include.inttypes
+    libc.include.libgen
     libc.include.limits
     libc.include.link
     libc.include.locale
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 8046679785e68..5ace968a739c8 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -1078,6 +1078,7 @@ list(APPEND TARGET_LIBM_ENTRYPOINTS
   libc.src.math.setpayloadbf16
   libc.src.math.setpayloadsigbf16
   libc.src.math.sqrtbf16
+  libc.src.math.tanbf16
   libc.src.math.truncbf16
   libc.src.math.totalorderbf16
   libc.src.math.totalordermagbf16
@@ -1372,6 +1373,7 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.stdlib.exit
     libc.src.stdlib.getenv
     libc.src.stdlib.setenv
+    libc.src.stdlib.unsetenv
     libc.src.stdlib.mbstowcs
     libc.src.stdlib.mbtowc
     libc.src.stdlib.mblen
diff --git a/libc/config/linux/x86_64/headers.txt b/libc/config/linux/x86_64/headers.txt
index 8b88ded158ebe..6cd16ecdae3e3 100644
--- a/libc/config/linux/x86_64/headers.txt
+++ b/libc/config/linux/x86_64/headers.txt
@@ -15,6 +15,7 @@ set(TARGET_PUBLIC_HEADERS
     libc.include.fenv
     libc.include.float
     libc.include.inttypes
+    libc.include.libgen
     libc.include.limits
     libc.include.link
     libc.include.locale
diff --git a/libc/config/windows/entrypoints.txt b/libc/config/windows/entrypoints.txt
index d7f66dc58e4db..3c68290675b11 100644
--- a/libc/config/windows/entrypoints.txt
+++ b/libc/config/windows/entrypoints.txt
@@ -393,6 +393,7 @@ list(APPEND TARGET_LIBM_ENTRYPOINTS
   libc.src.math.setpayloadbf16
   libc.src.math.setpayloadsigbf16
   libc.src.math.sqrtbf16
+  libc.src.math.tanbf16
   libc.src.math.truncbf16
   libc.src.math.totalorderbf16
   libc.src.math.totalordermagbf16
diff --git a/libc/docs/dev/builtin_compatibility.rst b/libc/docs/dev/builtin_compatibility.rst
new file mode 100644
index 0000000000000..4ea9cc9ba1115
--- /dev/null
+++ b/libc/docs/dev/builtin_compatibility.rst
@@ -0,0 +1,351 @@
+.. _builtin_compatibility:
+
+==============================
+GCC Compatibility of Builtins
+==============================
+
+LLVM-libc is written with Clang as the reference compiler but is expected to
+build under GCC as well.  Many headers reach for compiler intrinsics
+(``__builtin_*``) for fast paths and for IEEE-754 semantics.  Not every
+``__builtin_*`` exists in every compiler. a large family is Clang-only,
+and several others were added in specific GCC releases.  This page records
+which builtins the libc code base references today and the minimum GCC
+basepoint that ships each one.
+
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| Builtin                                | gcc-5   | gcc-6   | gcc-7   | gcc-8   | gcc-9   | gcc-10  | gcc-11  | gcc-12  | gcc-13  | gcc-14  | gcc-15  | gcc-16  | gcc-17  |
++========================================+=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+
+| ``__builtin_aarch64_get_fpcr``         |         | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_aarch64_get_fpsr``         |         | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_aarch64_set_fpcr``         |         | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_aarch64_set_fpsr``         |         | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_aarch64_wsr64``            |         |         |         |         |         |         |         |         |         |         | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_abs``                      | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_addc``                     |         |         |         |         |         |         |         |         |         |         | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_addcb``                    |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_addcl``                    |         |         |         |         |         |         |         |         |         |         | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_addcll``                   |         |         |         |         |         |         |         |         |         |         | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_addcs``                    |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_add_overflow``             |         | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_addressof``                |         |         |         | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_align_down``               |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_align_up``                 |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_alloca``                   | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_amdgcn_ldexp``             |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_amdgcn_ldexpf``            |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_amdgcn_s_sendmsg``         |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_amdgcn_s_sleep``           |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_amdgcn_workitem_id_x``     |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_arm_get_fpscr``            |         | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_arm_isb``                  |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_arm_set_fpscr``            |         | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_arm_wsr64``                |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_assume``                   |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_assume_aligned``           | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_bcmp``                     | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_bit_cast``                 |         |         |         |         |         |         |         | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_bswap16``                  | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_bswap32``                  | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_bswap64``                  | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_ceil``                     | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_ceilf``                    | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_clear_padding``            |         |         |         |         |         |         |         | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_clz``                      | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_clzg``                     |         |         |         |         |         |         |         |         |         |         | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_clzl``                     | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_clzll``                    | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_clzs``                     | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_complex``                  | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_convertvector``            |         |         |         |         |         | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_copysign``                 | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_copysignf``                | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_copysignf16``              |         |         |         |         |         | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_ctz``                      | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_ctzg``                     |         |         |         |         |         |         |         |         |         |         | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_ctzl``                     | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_ctzll``                    | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_ctzs``                     | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_elementwise_abs``          |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_elementwise_canonicalize`` |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_elementwise_ceil``         |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_elementwise_copysign``     |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_elementwise_floor``        |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_elementwise_fma``          |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_elementwise_fmod``         |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_elementwise_max``          |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_elementwise_min``          |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_elementwise_nearbyint``    |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_elementwise_rint``         |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_elementwise_round``        |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_elementwise_roundeven``    |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_elementwise_sqrt``         |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_elementwise_trunc``        |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_expect``                   | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_fabs``                     | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_fabsf``                    | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_fabsf16``                  |         |         |         |         |         |         | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_floor``                    | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_floorf``                   | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_flt_rounds``               |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_fma``                      | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_fmaf``                     | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_fmax``                     | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_fmaxf``                    | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_fmaxf16``                  |         |         |         |         |         |         |         |         | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_fmin``                     | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_fminf``                    | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_fminf16``                  |         |         |         |         |         |         |         |         | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_fmod``                     | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_fmodf``                    | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_fpclassify``               | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_frame_address``            | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_frexp``                    | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_frexpf``                   | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_func``                     |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_huge_val``                 | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_huge_valf``                | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_ia32_ldmxcsr``             | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_ia32_pause``               | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_ia32_stmxcsr``             | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_inff``                     | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_is_aligned``               |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_is_constant_evaluated``    |         |         |         |         |         | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_isfinite``                 | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_isinf``                    | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_isnan``                    | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_isnormal``                 | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_launder``                  |         |         |         | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_ldexp``                    | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_ldexpf``                   |         |         |         |         |         |         |         | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_masked_compress_store``    |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_masked_expand_load``       |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_masked_gather``            |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_masked_load``              |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_masked_scatter``           |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_masked_store``             |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_memcmp``                   | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_memcmp_inline``            |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_memcpy``                   | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_memcpy_inline``            |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_memmove``                  | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_memset``                   | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_memset_inline``            |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_mul_overflow``             |         | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_nanf``                     | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_nans``                     | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_nansf``                    | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_nansl``                    | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_nearbyint``                | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_nearbyintf``               | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_nondeterministic_value``   |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_nontemporal_store``        |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_offsetof``                 | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_popcount``                 | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_popcountg``                |         |         |         |         |         |         |         |         |         |         | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_popcountl``                | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_popcountll``               | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_prefetch``                 | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_readcyclecounter``         |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_readsteadycounter``        |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_reduce_add``               |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_reduce_and``               |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_reduce_max``               |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_reduce_min``               |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_reduce_mul``               |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_reduce_or``                |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_reduce_xor``               |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_remainder``                | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_remainderf``               | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_return_address``           | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_rint``                     | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_rintf``                    | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_round``                    | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_roundeven``                |         |         |         |         |         |         | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_roundevenf``               |         |         |         |         |         |         | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_roundf``                   | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_sadd_overflow``            |         | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_shufflevector``            |         |         |         |         |         |         |         |         | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_signbit``                  | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_snprintf``                 | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_sqrt``                     | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_sqrtf``                    | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_strlen``                   | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_subc``                     |         |         |         |         |         |         |         |         |         |         | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_subcb``                    |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_subcl``                    |         |         |         |         |         |         |         |         |         |         | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_subcll``                   |         |         |         |         |         |         |         |         |         |         | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_subcs``                    |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_sub_overflow``             |         | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_trap``                     | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_trunc``                    | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_truncf``                   | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_unreachable``              | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__builtin_vectorelements``           |         |         |         |         |         |         |         |         |         |         |         |         |         |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ``__has_builtin``                      |         |         |         |         |         | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| |
++----------------------------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
diff --git a/libc/docs/dev/index.rst b/libc/docs/dev/index.rst
index e6463712e3859..83d0956b7f1a9 100644
--- a/libc/docs/dev/index.rst
+++ b/libc/docs/dev/index.rst
@@ -20,5 +20,6 @@ Navigate to the links below for information on the respective topics:
    implementation_standard
    undefined_behavior
    printf_behavior
+   builtin_compatibility
    syscall_wrapper_refactor
    modular_format
diff --git a/libc/docs/headers/math/index.rst b/libc/docs/headers/math/index.rst
index c2ebc5031cef5..8f8f381054002 100644
--- a/libc/docs/headers/math/index.rst
+++ b/libc/docs/headers/math/index.rst
@@ -353,7 +353,7 @@ Higher Math Functions
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
 | sqrt      | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.7.10              | F.10.4.10                  |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
-| tan       | |check|          | |check|         |                        | |check|              |                        |                        | 7.12.4.7               | F.10.1.7                   |
+| tan       | |check|          | |check|         |                        | |check|              |                        | |check|                | 7.12.4.7               | F.10.1.7                   |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
 | tanh      | |check|          |                 |                        | |check|              |                        |                        | 7.12.5.6               | F.10.2.6                   |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
diff --git a/libc/include/stdlib.yaml b/libc/include/stdlib.yaml
index bf77d5d850ea0..93f61d541b073 100644
--- a/libc/include/stdlib.yaml
+++ b/libc/include/stdlib.yaml
@@ -376,6 +376,12 @@ functions:
     return_type: int
     arguments:
       - type: const char *
+  - name: unsetenv
+    standards:
+      - posix
+    return_type: int
+    arguments:
+      - type: const char *
   - name: wctomb
     standards:
       - stdc
diff --git a/libc/shared/math.h b/libc/shared/math.h
index f4b28e3aa139d..2e8a574e4d042 100644
--- a/libc/shared/math.h
+++ b/libc/shared/math.h
@@ -474,6 +474,7 @@
 #include "math/sqrtf16.h"
 #include "math/sqrtl.h"
 #include "math/tan.h"
+#include "math/tanbf16.h"
 #include "math/tanf.h"
 #include "math/tanf16.h"
 #include "math/tanhf.h"
diff --git a/libc/shared/math/tanbf16.h b/libc/shared/math/tanbf16.h
new file mode 100644
index 0000000000000..24006da13b4dd
--- /dev/null
+++ b/libc/shared/math/tanbf16.h
@@ -0,0 +1,23 @@
+//===-- Shared tanbf16 function ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SHARED_MATH_TANBF16_H
+#define LLVM_LIBC_SHARED_MATH_TANBF16_H
+
+#include "shared/libc_common.h"
+#include "src/__support/math/tanbf16.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace shared {
+
+using math::tanbf16;
+
+} // namespace shared
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SHARED_MATH_TANBF16_H
diff --git a/libc/src/CMakeLists.txt b/libc/src/CMakeLists.txt
index 56085c9632f59..62aa164b35ee7 100644
--- a/libc/src/CMakeLists.txt
+++ b/libc/src/CMakeLists.txt
@@ -14,11 +14,13 @@ add_subdirectory(netinet)
 add_subdirectory(stdbit)
 add_subdirectory(stdfix)
 add_subdirectory(stdio)
+# unistd must be added before stdlib because stdlib's environ_internal
+# target conditionally depends on unistd's environ target.
+add_subdirectory(unistd)
 add_subdirectory(stdlib)
 add_subdirectory(string)
 add_subdirectory(strings)
 add_subdirectory(time)
-add_subdirectory(unistd)
 add_subdirectory(wchar)
 add_subdirectory(wctype)
 
diff --git a/libc/src/__support/math/CMakeLists.txt b/libc/src/__support/math/CMakeLists.txt
index b57e82a036665..a57086a31dc03 100644
--- a/libc/src/__support/math/CMakeLists.txt
+++ b/libc/src/__support/math/CMakeLists.txt
@@ -5374,6 +5374,7 @@ add_header_library(
     libc.src.__support.uint128
     libc.include.llvm-libc-types.float128
 )
+
 add_header_library(
   sqrtl
   HDRS
@@ -5399,6 +5400,22 @@ add_header_library(
     libc.src.__support.macros.optimization
 )
 
+add_header_library(
+  tanbf16
+  HDRS
+    tanbf16.h
+  DEPENDS
+    .sincosf_utils
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.__support.FPUtil.cast
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.multiply_add
+    libc.src.__support.macros.optimization
+    libc.src.__support.FPUtil.bfloat16
+)
+
 add_header_library(
   tanf
   HDRS
diff --git a/libc/src/__support/math/tanbf16.h b/libc/src/__support/math/tanbf16.h
new file mode 100644
index 0000000000000..3dae149a4a643
--- /dev/null
+++ b/libc/src/__support/math/tanbf16.h
@@ -0,0 +1,83 @@
+//===-- Implementation of tanbf16 function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_MATH_TANBF16_H
+#define LLVM_LIBC_SRC___SUPPORT_MATH_TANBF16_H
+
+#include "hdr/errno_macros.h"
+#include "hdr/fenv_macros.h"
+#include "sincosf_utils.h"
+#include "src/__support/FPUtil/FEnvImpl.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/FPUtil/cast.h"
+#include "src/__support/FPUtil/multiply_add.h"
+#include "src/__support/macros/optimization.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+namespace math {
+
+LIBC_INLINE bfloat16 tanbf16(bfloat16 x) {
+
+  using FPBits = fputil::FPBits<bfloat16>;
+  FPBits xbits(x);
+
+  uint16_t x_u = xbits.uintval();
+  uint16_t x_abs = x_u & 0x7fff;
+  float xf = x;
+
+  // NaN or -/+ INF
+  if (x_abs >= 0x7F80) {
+    // NaN
+    if (xbits.is_nan()) {
+      if (xbits.is_signaling_nan()) {
+        fputil::raise_except_if_required(FE_INVALID);
+        return FPBits::quiet_nan().get_val();
+      }
+      return x;
+    }
+    //|x| is +/- INF
+    fputil::set_errno_if_required(EDOM);
+    fputil::raise_except_if_required(FE_INVALID);
+    return x + FPBits::quiet_nan().get_val();
+  }
+
+  // Through Exhaustive testing -
+  // The last value where tan(x) ~ x is 0x3db8
+  if (LIBC_UNLIKELY(x_abs <= 0x3db8)) {
+    // |x| = {0}
+    if (LIBC_UNLIKELY(x_abs == 0)) {
+      return x;
+    }
+    return fputil::cast<bfloat16>(fputil::multiply_add(xf, 0x1.0p-11f, xf));
+  }
+
+  double xd = static_cast<double>(xf);
+  uint32_t x_abs_f = fputil::FPBits<float>(xf).uintval() & 0x7fffffff;
+  double sin_k, cos_k, sin_y, cosm1_y;
+
+  // TODO: Use bfloat16 version for inv_trigf_utils_internals after they are
+  // available
+  // Tracking issue :
+  // https://github.com/llvm/llvm-project/issues/202079
+  sincosf_utils_internal::sincosf_eval(xd, x_abs_f, sin_k, cos_k, sin_y,
+                                       cosm1_y);
+
+  return fputil::cast<bfloat16>(
+      fputil::multiply_add(sin_y, cos_k,
+                           fputil::multiply_add(cosm1_y, sin_k, sin_k)) /
+      fputil::multiply_add(sin_y, -sin_k,
+                           fputil::multiply_add(cosm1_y, cos_k, cos_k)));
+}
+
+} // namespace math
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_MATH_TANBF16_H
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index dcfc8183d716a..4823ceefd7475 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -587,6 +587,7 @@ add_math_entrypoint_object(sqrtf128)
 add_math_entrypoint_object(sqrtbf16)
 
 add_math_entrypoint_object(tan)
+add_math_entrypoint_object(tanbf16)
 add_math_entrypoint_object(tanf)
 add_math_entrypoint_object(tanf16)
 
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index 770da1404a95b..5a5bd84213a55 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -415,6 +415,16 @@ add_entrypoint_object(
     libc.src.errno.errno
 )
 
+add_entrypoint_object(
+  tanbf16
+  SRCS
+    tanbf16.cpp
+  HDRS
+    ../tanbf16.h
+  DEPENDS
+    libc.src.__support.math.tanbf16
+)
+
 add_entrypoint_object(
   tanf
   SRCS
diff --git a/libc/src/math/generic/tanbf16.cpp b/libc/src/math/generic/tanbf16.cpp
new file mode 100644
index 0000000000000..990272fdcd677
--- /dev/null
+++ b/libc/src/math/generic/tanbf16.cpp
@@ -0,0 +1,16 @@
+//===-- Implementation for tanbf16(x) function ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception.
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/tanbf16.h"
+#include "src/__support/math/tanbf16.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, tanbf16, (bfloat16 x)) { return math::tanbf16(x); }
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/tanbf16.h b/libc/src/math/tanbf16.h
new file mode 100644
index 0000000000000..034167d3bb9c2
--- /dev/null
+++ b/libc/src/math/tanbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for tanbf16 -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_TANBF16_H
+#define LLVM_LIBC_SRC_MATH_TANBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 tanbf16(bfloat16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_TANBF16_H
diff --git a/libc/src/stdlib/CMakeLists.txt b/libc/src/stdlib/CMakeLists.txt
index 4593288881c7d..1e26588ad7305 100644
--- a/libc/src/stdlib/CMakeLists.txt
+++ b/libc/src/stdlib/CMakeLists.txt
@@ -77,6 +77,24 @@ if((${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU") AND
       "-fdisable-tree-waccess3")
 endif()
 
+set(environ_internal_deps
+  libc.config.app_h
+  libc.hdr.types.size_t
+  libc.src.__support.alloc_checker
+  libc.src.__support.CPP.new
+  libc.src.__support.CPP.optional
+  libc.src.__support.CPP.string_view
+  libc.src.__support.macros.attributes
+  libc.src.__support.macros.config
+  libc.src.string.memory_utils.inline_memcpy
+)
+set(environ_internal_compile_options ${environ_internal_gcc12_workaround})
+
+if("libc.src.unistd.environ" IN_LIST TARGET_LLVMLIBC_ENTRYPOINTS)
+  list(APPEND environ_internal_deps libc.src.unistd.environ)
+  list(APPEND environ_internal_compile_options "-DLIBC_COPT_SUPPORT_ENVIRON")
+endif()
+
 add_object_library(
   environ_internal
   SRCS
@@ -84,16 +102,9 @@ add_object_library(
   HDRS
     environ_internal.h
   COMPILE_OPTIONS
-    ${environ_internal_gcc12_workaround}
+    ${environ_internal_compile_options}
   DEPENDS
-    libc.config.app_h
-    libc.hdr.types.size_t
-    libc.src.__support.CPP.new
-    libc.src.__support.CPP.optional
-    libc.src.__support.CPP.string_view
-    libc.src.__support.macros.attributes
-    libc.src.__support.macros.config
-    libc.src.string.memory_utils.inline_memcpy
+    ${environ_internal_deps}
 )
 
 add_entrypoint_object(
@@ -677,6 +688,13 @@ add_entrypoint_object(
     .${LIBC_TARGET_OS}.setenv
 )
 
+add_entrypoint_object(
+  unsetenv
+  ALIAS
+  DEPENDS
+    .${LIBC_TARGET_OS}.unsetenv
+)
+
 if(LIBC_TARGET_OS_IS_BAREMETAL OR LIBC_TARGET_OS_IS_GPU)
   add_entrypoint_object(
     malloc
diff --git a/libc/src/stdlib/environ_internal.cpp b/libc/src/stdlib/environ_internal.cpp
index e9da17d028575..d7ef081e7e2b1 100644
--- a/libc/src/stdlib/environ_internal.cpp
+++ b/libc/src/stdlib/environ_internal.cpp
@@ -18,6 +18,9 @@
 #include "src/__support/alloc-checker.h"
 #include "src/__support/macros/config.h"
 #include "src/string/memory_utils/inline_memcpy.h"
+#ifdef LIBC_COPT_SUPPORT_ENVIRON
+#include "src/unistd/environ.h"
+#endif
 
 namespace LIBC_NAMESPACE_DECL {
 namespace internal {
@@ -142,6 +145,9 @@ bool EnvironmentManager::ensure_capacity(size_t needed) {
 
     // Update the global environ pointer.
     app.env_ptr = reinterpret_cast<uintptr_t *>(storage);
+#ifdef LIBC_COPT_SUPPORT_ENVIRON
+    environ = storage;
+#endif
 
     return true;
   }
@@ -168,6 +174,9 @@ bool EnvironmentManager::ensure_capacity(size_t needed) {
 
   // Update the global environ pointer.
   app.env_ptr = reinterpret_cast<uintptr_t *>(storage);
+#ifdef LIBC_COPT_SUPPORT_ENVIRON
+  environ = storage;
+#endif
 
   return true;
 }
@@ -221,5 +230,40 @@ int EnvironmentManager::set(cpp::string_view name, cpp::string_view value,
   return 0;
 }
 
+int EnvironmentManager::unset(cpp::string_view name) {
+  cpp::optional<size_t> idx = find_var(name);
+  if (!idx)
+    return 0; // Variable not found; POSIX defines this as success.
+
+  // Transition to managed storage so we can modify the array and track
+  // ownership correctly.
+  if (!ensure_capacity(count))
+    return -1;
+
+  char **env_array = get_array();
+
+  // Loop to remove all instances of the variable (e.g. duplicates from execve).
+  while (idx) {
+    size_t i = *idx;
+    if (ownership[i].can_free())
+      delete[] env_array[i];
+
+    // Compact: shift remaining entries left to fill the gap.
+    // Shifting elements preserves the order of environment variables, which is
+    // desirable to maintain consistency with getenv resolution order and
+    // typical environ iteration behavior.
+    for (size_t j = i; j < count - 1; j++) {
+      env_array[j] = env_array[j + 1];
+      ownership[j] = ownership[j + 1];
+    }
+    count--;
+    env_array[count] = nullptr;
+
+    idx = find_var(name);
+  }
+
+  return 0;
+}
+
 } // namespace internal
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/stdlib/environ_internal.h b/libc/src/stdlib/environ_internal.h
index d902354f421cf..f16d841779734 100644
--- a/libc/src/stdlib/environ_internal.h
+++ b/libc/src/stdlib/environ_internal.h
@@ -129,6 +129,11 @@ class EnvironmentManager {
   // Returns 0 on success, -1 on allocation failure (caller should set
   // errno to ENOMEM).
   int set(cpp::string_view name, cpp::string_view value, bool overwrite);
+
+  // Remove a variable by name. Frees the string if we own it, then
+  // compacts the array. Returns 0 on success (including if the variable
+  // was not found), -1 on allocation failure during array transition.
+  int unset(cpp::string_view name);
 };
 
 } // namespace internal
diff --git a/libc/src/stdlib/linux/CMakeLists.txt b/libc/src/stdlib/linux/CMakeLists.txt
index aedcd3f11bf38..06af6f046afa1 100644
--- a/libc/src/stdlib/linux/CMakeLists.txt
+++ b/libc/src/stdlib/linux/CMakeLists.txt
@@ -23,3 +23,18 @@ add_entrypoint_object(
     libc.src.__support.macros.null_check
     libc.src.stdlib.environ_internal
 )
+
+add_entrypoint_object(
+  unsetenv
+  SRCS
+    unsetenv.cpp
+  HDRS
+    ../unsetenv.h
+  DEPENDS
+    libc.src.__support.CPP.string_view
+    libc.src.__support.common
+    libc.src.__support.libc_errno
+    libc.src.__support.macros.config
+    libc.src.__support.macros.null_check
+    libc.src.stdlib.environ_internal
+)
diff --git a/libc/src/stdlib/linux/unsetenv.cpp b/libc/src/stdlib/linux/unsetenv.cpp
new file mode 100644
index 0000000000000..538242ed29fca
--- /dev/null
+++ b/libc/src/stdlib/linux/unsetenv.cpp
@@ -0,0 +1,43 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Implementation of the POSIX unsetenv function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "src/stdlib/unsetenv.h"
+#include "src/__support/CPP/string_view.h"
+#include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
+#include "src/stdlib/environ_internal.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, unsetenv, (const char *name)) {
+  LIBC_CRASH_ON_NULLPTR(name);
+
+  cpp::string_view name_view(name);
+
+  // POSIX: name must not be empty or contain '='.
+  if (name_view.empty() ||
+      name_view.find_first_of('=') != cpp::string_view::npos) {
+    libc_errno = EINVAL;
+    return -1;
+  }
+
+  int result = internal::EnvironmentManager::get_instance().unset(name_view);
+  if (result != 0)
+    libc_errno = ENOMEM;
+
+  return result;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/stdlib/unsetenv.h b/libc/src/stdlib/unsetenv.h
new file mode 100644
index 0000000000000..b705d1b481aef
--- /dev/null
+++ b/libc/src/stdlib/unsetenv.h
@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Declaration of the POSIX unsetenv function.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STDLIB_UNSETENV_H
+#define LLVM_LIBC_SRC_STDLIB_UNSETENV_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int unsetenv(const char *name);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_STDLIB_UNSETENV_H
diff --git a/libc/src/wchar/wcslcat.cpp b/libc/src/wchar/wcslcat.cpp
index eb318e066f7a0..456aac4543e41 100644
--- a/libc/src/wchar/wcslcat.cpp
+++ b/libc/src/wchar/wcslcat.cpp
@@ -21,8 +21,10 @@ LLVM_LIBC_FUNCTION(size_t, wcslcat,
                     size_t dstsize)) {
   const size_t dstlen = internal::string_length(dst);
   const size_t srclen = internal::string_length(src);
+  if (dstlen >= dstsize)
+    return dstsize + srclen;
   int limit = static_cast<int>(dstsize - dstlen - 1);
-  size_t returnval = (dstsize < dstlen ? dstsize : dstlen) + srclen;
+  size_t returnval = dstlen + srclen;
   if (limit < 0)
     return returnval;
   int i = 0;
diff --git a/libc/src/wchar/wcsncat.cpp b/libc/src/wchar/wcsncat.cpp
index 62595b4b5418c..985fb5f1cbace 100644
--- a/libc/src/wchar/wcsncat.cpp
+++ b/libc/src/wchar/wcsncat.cpp
@@ -21,7 +21,7 @@ LLVM_LIBC_FUNCTION(wchar_t *, wcsncat,
                     size_t n)) {
   size_t size = internal::string_length(s1);
   size_t i = 0;
-  for (; s2[i] && i < n; ++i)
+  for (; i < n && s2[i]; ++i)
     s1[size + i] = s2[i];
   // Appending null character to the end of the result.
   s1[size + i] = L'\0';
diff --git a/libc/test/integration/src/stdlib/CMakeLists.txt b/libc/test/integration/src/stdlib/CMakeLists.txt
index 2e0389f9a60d3..789f9aded9570 100644
--- a/libc/test/integration/src/stdlib/CMakeLists.txt
+++ b/libc/test/integration/src/stdlib/CMakeLists.txt
@@ -30,6 +30,22 @@ if(${LIBC_TARGET_OS} STREQUAL "linux")
       libc.src.string.strcmp
   )
 
+  add_integration_test(
+    unsetenv_test
+    SUITE
+      stdlib-integration-tests
+    SRCS
+      unsetenv_test.cpp
+    DEPENDS
+      libc.src.__support.alloc_checker
+      libc.src.stdlib.getenv
+      libc.src.stdlib.setenv
+      libc.src.stdlib.unsetenv
+      libc.src.string.memory_utils.inline_memcpy
+      libc.src.string.strcmp
+      libc.src.unistd.environ
+  )
+
   add_integration_test(
     abort_test
     SUITE
diff --git a/libc/test/integration/src/stdlib/unsetenv_test.cpp b/libc/test/integration/src/stdlib/unsetenv_test.cpp
new file mode 100644
index 0000000000000..53341940702ed
--- /dev/null
+++ b/libc/test/integration/src/stdlib/unsetenv_test.cpp
@@ -0,0 +1,176 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Integration tests for unsetenv.
+///
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/alloc-checker.h"
+#include "src/stdlib/environ_internal.h"
+#include "src/stdlib/getenv.h"
+#include "src/stdlib/setenv.h"
+#include "src/stdlib/unsetenv.h"
+#include "src/string/memory_utils/inline_memcpy.h"
+#include "src/string/strcmp.h"
+#include "src/unistd/environ.h"
+
+#include "test/IntegrationTest/test.h"
+
+#include <errno.h>
+
+namespace LIBC_NAMESPACE {
+
+TEST_MAIN([[maybe_unused]] int argc, [[maybe_unused]] char **argv,
+          [[maybe_unused]] char **envp) {
+  // Test: Remove a variable set by setenv
+  {
+    ASSERT_EQ(setenv("UNSET_VAR", "value", 1), 0);
+    ASSERT_TRUE(getenv("UNSET_VAR") != nullptr);
+
+    ASSERT_EQ(unsetenv("UNSET_VAR"), 0);
+    ASSERT_TRUE(getenv("UNSET_VAR") == nullptr);
+  }
+
+  // Test: Unset non-existent variable succeeds
+  {
+    ASSERT_EQ(unsetenv("DOES_NOT_EXIST"), 0);
+  }
+
+  // Test: Empty name sets errno to EINVAL
+  {
+    errno = 0;
+    ASSERT_EQ(unsetenv(""), -1);
+    ASSERT_ERRNO_EQ(EINVAL);
+  }
+
+  // Test: Name with '=' sets errno to EINVAL
+  {
+    errno = 0;
+    ASSERT_EQ(unsetenv("BAD=NAME"), -1);
+    ASSERT_ERRNO_EQ(EINVAL);
+  }
+
+  // Test: Unset then re-set
+  {
+    ASSERT_EQ(setenv("REUSE_VAR", "first", 1), 0);
+    ASSERT_EQ(strcmp(getenv("REUSE_VAR"), "first"), 0);
+
+    ASSERT_EQ(unsetenv("REUSE_VAR"), 0);
+    ASSERT_TRUE(getenv("REUSE_VAR") == nullptr);
+
+    ASSERT_EQ(setenv("REUSE_VAR", "second", 1), 0);
+    ASSERT_EQ(strcmp(getenv("REUSE_VAR"), "second"), 0);
+  }
+
+  // Test: Unset multiple variables
+  {
+    ASSERT_EQ(setenv("MULTI_A", "a", 1), 0);
+    ASSERT_EQ(setenv("MULTI_B", "b", 1), 0);
+    ASSERT_EQ(setenv("MULTI_C", "c", 1), 0);
+
+    ASSERT_EQ(unsetenv("MULTI_B"), 0);
+
+    ASSERT_TRUE(getenv("MULTI_A") != nullptr);
+    ASSERT_TRUE(getenv("MULTI_B") == nullptr);
+    ASSERT_TRUE(getenv("MULTI_C") != nullptr);
+
+    ASSERT_EQ(strcmp(getenv("MULTI_A"), "a"), 0);
+    ASSERT_EQ(strcmp(getenv("MULTI_C"), "c"), 0);
+  }
+
+  // Test: Unset same variable twice is harmless
+  {
+    ASSERT_EQ(setenv("DOUBLE_UNSET", "val", 1), 0);
+    ASSERT_EQ(unsetenv("DOUBLE_UNSET"), 0);
+    ASSERT_EQ(unsetenv("DOUBLE_UNSET"), 0);
+    ASSERT_TRUE(getenv("DOUBLE_UNSET") == nullptr);
+  }
+
+  // Test: environ is updated and does NOT contain the variable
+  {
+    ASSERT_EQ(setenv("ENV_CHECK", "val", 1), 0);
+    // Verify it is in environ
+    bool found = false;
+    for (char **env = environ; *env != nullptr; ++env) {
+      if (strcmp(*env, "ENV_CHECK=val") == 0) {
+        found = true;
+        break;
+      }
+    }
+    ASSERT_TRUE(found);
+
+    // Now unset it
+    ASSERT_EQ(unsetenv("ENV_CHECK"), 0);
+
+    // Verify it is NOT in environ
+    found = false;
+    for (char **env = environ; *env != nullptr; ++env) {
+      bool match = true;
+      const char *prefix = "ENV_CHECK=";
+      for (size_t i = 0; i < 10; ++i) {
+        if ((*env)[i] == '\0' || (*env)[i] != prefix[i]) {
+          match = false;
+          break;
+        }
+      }
+      if (match) {
+        found = true;
+        break;
+      }
+    }
+    ASSERT_FALSE(found);
+  }
+
+  // Test: Unset removes all duplicate instances of a variable
+  {
+    ASSERT_EQ(setenv("DUP_VAL1", "1", 1), 0);
+    ASSERT_EQ(setenv("DUP_VAL2", "2", 1), 0);
+
+    AllocChecker ac;
+    char *d1 = new (ac) char[14];
+    ASSERT_TRUE(ac);
+    const char *s1 = "DUP_VAR=first";
+    inline_memcpy(d1, s1, 14);
+
+    char *d2 = new (ac) char[15];
+    ASSERT_TRUE(ac);
+    const char *s2 = "DUP_VAR=second";
+    inline_memcpy(d2, s2, 15);
+
+    internal::EnvironmentManager &mgr =
+        internal::EnvironmentManager::get_instance();
+    char **env_array = mgr.begin();
+    size_t count = mgr.size();
+
+    size_t idx1 = count;
+    size_t idx2 = count;
+    for (size_t i = 0; i < count; ++i) {
+      cpp::string_view curr(env_array[i]);
+      if (curr.starts_with("DUP_VAL1="))
+        idx1 = i;
+      if (curr.starts_with("DUP_VAL2="))
+        idx2 = i;
+    }
+    ASSERT_TRUE(idx1 < count);
+    ASSERT_TRUE(idx2 < count);
+
+    delete[] env_array[idx1];
+    delete[] env_array[idx2];
+
+    env_array[idx1] = d1;
+    env_array[idx2] = d2;
+
+    ASSERT_EQ(unsetenv("DUP_VAR"), 0);
+    ASSERT_TRUE(getenv("DUP_VAR") == nullptr);
+  }
+
+  return 0;
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/test/shared/CMakeLists.txt b/libc/test/shared/CMakeLists.txt
index 17e1570f1b19a..9b990dd5a5c84 100644
--- a/libc/test/shared/CMakeLists.txt
+++ b/libc/test/shared/CMakeLists.txt
@@ -479,6 +479,7 @@ add_fp_unittest(
     libc.src.__support.math.sqrtf
     libc.src.__support.math.sqrtl
     libc.src.__support.math.tan
+    libc.src.__support.math.tanbf16
     libc.src.__support.math.tanf
     libc.src.__support.math.tanf16
     libc.src.__support.math.tanhf
diff --git a/libc/test/shared/shared_math_test.cpp b/libc/test/shared/shared_math_test.cpp
index e26532773c69b..5877729ae7931 100644
--- a/libc/test/shared/shared_math_test.cpp
+++ b/libc/test/shared/shared_math_test.cpp
@@ -893,5 +893,6 @@ TEST(LlvmLibcSharedMathTest, AllBFloat16) {
   EXPECT_FP_EQ(bfloat16(0.0), LIBC_NAMESPACE::shared::roundbf16(bfloat16(0.0)));
   EXPECT_FP_EQ(bfloat16(0.0),
                LIBC_NAMESPACE::shared::roundevenbf16(bfloat16(0.0)));
+  EXPECT_FP_EQ(bfloat16(0.0), LIBC_NAMESPACE::shared::tanbf16(bfloat16(0.0)));
   EXPECT_FP_EQ(bfloat16(0.0), LIBC_NAMESPACE::shared::truncbf16(bfloat16(0.0)));
 }
diff --git a/libc/test/src/math/CMakeLists.txt b/libc/test/src/math/CMakeLists.txt
index 1e27724fe0a60..31f6468e01110 100644
--- a/libc/test/src/math/CMakeLists.txt
+++ b/libc/test/src/math/CMakeLists.txt
@@ -201,6 +201,19 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  tanbf16_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    tanbf16_test.cpp
+  DEPENDS
+    libc.src.math.tanbf16
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.bfloat16
+)
+
 add_fp_unittest(
   tanf_test
   NEED_MPFR
diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt
index b763cdb43b31c..73d2265322c4d 100644
--- a/libc/test/src/math/smoke/CMakeLists.txt
+++ b/libc/test/src/math/smoke/CMakeLists.txt
@@ -110,6 +110,19 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  tanbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    tanbf16_test.cpp
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.src.math.tanbf16
+    libc.hdr.fenv_macros
+    libc.src.__support.FPUtil.bfloat16
+)
+
 add_fp_unittest(
   tanf_test
   SUITE
diff --git a/libc/test/src/math/smoke/tanbf16_test.cpp b/libc/test/src/math/smoke/tanbf16_test.cpp
new file mode 100644
index 0000000000000..4e760b4a747e4
--- /dev/null
+++ b/libc/test/src/math/smoke/tanbf16_test.cpp
@@ -0,0 +1,41 @@
+//===-- Unittests for tanbf16 ---------------------------------------------===//
+//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception.
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/errno_macros.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/tanbf16.h"
+#include "test/UnitTest/FEnvSafeTest.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+class LlvmLibcTanBf16Test : public LIBC_NAMESPACE::testing::FEnvSafeTest {
+  DECLARE_SPECIAL_CONSTANTS(bfloat16)
+public:
+  void test_special_numbers() {
+    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::tanbf16(aNaN));
+    EXPECT_MATH_ERRNO(0);
+
+    EXPECT_FP_EQ_WITH_EXCEPTION_ALL_ROUNDING(
+        aNaN, LIBC_NAMESPACE::tanbf16(sNaN), FE_INVALID);
+    EXPECT_MATH_ERRNO(0);
+
+    EXPECT_FP_EQ_ALL_ROUNDING(zero, LIBC_NAMESPACE::tanbf16(zero));
+    EXPECT_MATH_ERRNO(0);
+
+    EXPECT_FP_EQ_ALL_ROUNDING(neg_zero, LIBC_NAMESPACE::tanbf16(neg_zero));
+    EXPECT_MATH_ERRNO(0);
+
+    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::tanbf16(inf));
+    EXPECT_MATH_ERRNO(EDOM);
+
+    EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::tanbf16(neg_inf));
+    EXPECT_MATH_ERRNO(EDOM);
+  }
+};
+TEST_F(LlvmLibcTanBf16Test, SpecialNumbers) { test_special_numbers(); }
diff --git a/libc/test/src/math/tanbf16_test.cpp b/libc/test/src/math/tanbf16_test.cpp
new file mode 100644
index 0000000000000..848e07debe18d
--- /dev/null
+++ b/libc/test/src/math/tanbf16_test.cpp
@@ -0,0 +1,43 @@
+//===-- Exhaustive test for tanbf16 ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/tanbf16.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+#include "utils/MPFRWrapper/MPFRUtils.h"
+
+using LlvmLibcTanBf16Test = LIBC_NAMESPACE::testing::FPTest<bfloat16>;
+
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+
+// Range: [0, Inf]
+static constexpr uint16_t POS_START = 0x0000U;
+static constexpr uint16_t POS_STOP = 0x7f80U;
+
+// Range: [-Inf, 0]
+static constexpr uint16_t NEG_START = 0x8000U;
+static constexpr uint16_t NEG_STOP = 0xff80U;
+
+TEST_F(LlvmLibcTanBf16Test, PositiveRange) {
+  for (uint16_t v = POS_START; v <= POS_STOP; ++v) {
+    bfloat16 x = FPBits(v).get_val();
+
+    EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Tan, x,
+                                   LIBC_NAMESPACE::tanbf16(x), 0.5);
+  }
+}
+
+TEST_F(LlvmLibcTanBf16Test, NegativeRange) {
+  for (uint16_t v = NEG_START; v <= NEG_STOP; ++v) {
+    bfloat16 x = FPBits(v).get_val();
+
+    EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Tan, x,
+                                   LIBC_NAMESPACE::tanbf16(x), 0.5);
+  }
+}
diff --git a/libc/test/src/wchar/wcslcat_test.cpp b/libc/test/src/wchar/wcslcat_test.cpp
index bad37496384e2..564a5c60006e6 100644
--- a/libc/test/src/wchar/wcslcat_test.cpp
+++ b/libc/test/src/wchar/wcslcat_test.cpp
@@ -55,3 +55,15 @@ TEST(LlvmLibcWCSLCatTest, SmallerNoOverwriteAfter0) {
   ASSERT_TRUE(dst[7] == L'\0');
   ASSERT_EQ(res, size_t(4));
 }
+
+TEST(LlvmLibcWCSLCatTest, DstsizeLessThanDstlen) {
+  const wchar_t *src = L"d";
+  wchar_t dst[4]{L"abc"};
+  // Should return src length + dst size
+  size_t res = LIBC_NAMESPACE::wcslcat(dst, src, 2);
+  ASSERT_TRUE(dst[0] == L'a');
+  ASSERT_TRUE(dst[1] == L'b');
+  ASSERT_TRUE(dst[2] == L'c');
+  ASSERT_TRUE(dst[3] == L'\0');
+  ASSERT_EQ(res, size_t(3));
+}
diff --git a/libc/test/src/wchar/wcsncat_test.cpp b/libc/test/src/wchar/wcsncat_test.cpp
index 47359f88cec9e..0795685713e00 100644
--- a/libc/test/src/wchar/wcsncat_test.cpp
+++ b/libc/test/src/wchar/wcsncat_test.cpp
@@ -80,3 +80,15 @@ TEST(LlvmLibcWCSNCatTest, NonEmptyDest) {
   ASSERT_TRUE(dest[3] == L'\0');
   ASSERT_TRUE(dest[4] == L'Z');
 }
+
+TEST(LlvmLibcWCSNCatTest, OOBAccess) {
+  wchar_t dest[4] = {L'\0'};
+  wchar_t src[3] = {L'a', L'b', L'c'};
+
+  // Checking the bound
+  LIBC_NAMESPACE::wcsncat(dest, src, 3);
+  ASSERT_TRUE(dest[0] == L'a');
+  ASSERT_TRUE(dest[1] == L'b');
+  ASSERT_TRUE(dest[2] == L'c');
+  ASSERT_TRUE(dest[3] == L'\0');
+}
diff --git a/libcxx/include/__algorithm/simd_utils.h b/libcxx/include/__algorithm/simd_utils.h
index f73c9ea4b6ea7..c47f79ac7f1dd 100644
--- a/libcxx/include/__algorithm/simd_utils.h
+++ b/libcxx/include/__algorithm/simd_utils.h
@@ -124,15 +124,6 @@ template <class _VecT, size_t _Np, class _Iter>
     return _VecT{__iter[_LoadIndices]..., ((void)_ZeroIndices, 0)...};
   }(make_index_sequence<_Np>{}, make_index_sequence<__simd_vector_size_v<_VecT> - _Np>{});
 }
-
-// Create a vector where every elements is __val
-template <class _VecT>
-[[__nodiscard__]] _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _VecT
-__broadcast(__simd_vector_underlying_type_t<_VecT> __val) {
-  return [&]<std::size_t... _Indices>(index_sequence<_Indices...>) {
-    return _VecT{((void)_Indices, __val)...};
-  }(make_index_sequence<__simd_vector_size_v<_VecT>>());
-}
 _LIBCPP_DIAGNOSTIC_POP
 
 template <class _Tp, size_t _Np>
diff --git a/libcxx/include/__locale_dir/num.h b/libcxx/include/__locale_dir/num.h
index 8af427ca37dbe..5b3b917384121 100644
--- a/libcxx/include/__locale_dir/num.h
+++ b/libcxx/include/__locale_dir/num.h
@@ -96,9 +96,8 @@ struct __num_get : protected __num_get_base {
       _LIBCPP_DIAGNOSTIC_PUSH
       _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wpsabi")
       using __vec   = __simd_vector<char, 32>;
-      __vec __chars = std::__broadcast<__vec>(__val);
       __vec __cmp   = std::__partial_load<__vec, __int_chr_cnt>(__atoms);
-      auto __res    = __chars == __cmp;
+      auto __res    = __vec(__val) == __cmp;
       if (std::__none_of(__res))
         return __int_chr_cnt;
       return std::min(__int_chr_cnt, std::__find_first_set(__res));
diff --git a/libcxx/include/__memory/shared_ptr.h b/libcxx/include/__memory/shared_ptr.h
index 8d12e395bb86c..c1d4222f195dc 100644
--- a/libcxx/include/__memory/shared_ptr.h
+++ b/libcxx/include/__memory/shared_ptr.h
@@ -90,7 +90,7 @@ class weak_ptr;
 
 template <class _Tp, class _Dp, class _Alloc>
 class _LIBCPP_HIDE_STRUCT_FROM_ABI __shared_ptr_pointer final : public __shared_weak_count {
-  using __alloc_t = __allocator_traits_rebind_t<_Alloc, __shared_ptr_pointer>;
+  using __alloc_t _LIBCPP_NODEBUG = __allocator_traits_rebind_t<_Alloc, __shared_ptr_pointer>;
 
   _LIBCPP_NO_UNIQUE_ADDRESS __alloc_t __alloc_;
   _LIBCPP_NO_UNIQUE_ADDRESS _Dp __deleter_;
@@ -158,7 +158,7 @@ struct _LIBCPP_HIDE_STRUCT_FROM_ABI __shared_ptr_emplace_for_overwrite : __share
   using __alloc_t _LIBCPP_NODEBUG    = __allocator_traits_rebind_t<_Alloc, __shared_ptr_emplace_for_overwrite>;
   using __value_type _LIBCPP_NODEBUG = __remove_cv_t<_Tp>;
 
-  explicit __shared_ptr_emplace_for_overwrite(_Alloc __a) : __alloc_(std::move(__a)) {}
+  _LIBCPP_HIDE_FROM_ABI explicit __shared_ptr_emplace_for_overwrite(_Alloc __a) : __alloc_(std::move(__a)) {}
 
   _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __on_zero_shared() _NOEXCEPT override { __value_.~__value_type(); }
   _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __on_zero_shared_weak() _NOEXCEPT override {
@@ -168,7 +168,7 @@ struct _LIBCPP_HIDE_STRUCT_FROM_ABI __shared_ptr_emplace_for_overwrite : __share
     __alloc_traits::deallocate(__tmp, pointer_traits<typename __alloc_traits::pointer>::pointer_to(*this), 1);
   }
 
-  __value_type* __get_elem() _NOEXCEPT { return std::addressof(__value_); }
+  _LIBCPP_HIDE_FROM_ABI __value_type* __get_elem() _NOEXCEPT { return std::addressof(__value_); }
 
 private:
   _LIBCPP_NO_UNIQUE_ADDRESS __alloc_t __alloc_;
@@ -602,12 +602,12 @@ class _LIBCPP_SHARED_PTR_TRIVIAL_ABI shared_ptr {
 
   template <class _Vp, class = void>
   struct __get_shared_from_this {
-    using type = __nat;
+    using type _LIBCPP_NODEBUG = __nat;
   };
 
   template <class _Vp>
   struct __get_shared_from_this<_Vp, __void_t<decltype(__get_shared_from_this_impl(static_cast<_Vp*>(nullptr)))> > {
-    using type = const enable_shared_from_this<typename decltype(__get_shared_from_this_impl(
+    using type _LIBCPP_NODEBUG = const enable_shared_from_this<typename decltype(__get_shared_from_this_impl(
         static_cast<_Vp*>(nullptr)))::type>*;
   };
 
@@ -616,7 +616,7 @@ class _LIBCPP_SHARED_PTR_TRIVIAL_ABI shared_ptr {
       class _OrigPtr,
       __enable_if_t<!is_convertible<_OrigPtr, typename __get_shared_from_this<__remove_pointer_t<_Yp> >::type>::value,
                     int> = 0>
-  void __enable_weak_this(_Yp, _OrigPtr) {}
+  _LIBCPP_HIDE_FROM_ABI void __enable_weak_this(_Yp, _OrigPtr) {}
 
   template <class, class _Yp>
   struct __shared_ptr_default_delete : default_delete<_Yp> {};
diff --git a/libsycl/docs/index.rst b/libsycl/docs/index.rst
index 4e92a219163ca..a5d657f3e6404 100644
--- a/libsycl/docs/index.rst
+++ b/libsycl/docs/index.rst
@@ -126,7 +126,7 @@ TODO for added SYCL classes
   * handle sub devices once they are implemented (blocked by liboffload support)
 
 * ``event``: get_wait_list, get_info, get_profiling_info, wait_and_throw & default ctor are not implemented
-* ``range``, ``id`` - to add operators
+* ``range``, ``id`` - __SYCL_DISABLE_ID_TO_INT_CONV__ and __SYCL_ASSUME_ID_RANGE optimizations are not implemented
 * general opens:
 
   * define a way to report errors from object dtors
diff --git a/libsycl/include/sycl/__impl/index_space_classes.hpp b/libsycl/include/sycl/__impl/index_space_classes.hpp
index 823dcea062d15..8f99d03d004ee 100644
--- a/libsycl/include/sycl/__impl/index_space_classes.hpp
+++ b/libsycl/include/sycl/__impl/index_space_classes.hpp
@@ -27,36 +27,40 @@ namespace detail {
 
 class Builder;
 
+template <typename N, typename T>
+using IntegralType = std::enable_if_t<std::is_integral_v<N>, T>;
+
 /// Helper class for dimensions data management.
-template <int Dimensions = 1> class RawArray {
+template <typename Derived, int Dimensions> class IndexSpaceBase {
   static_assert(Dimensions >= 1 && Dimensions <= 3,
-                "RawArray can only be 1, 2, or 3 Dimensional.");
+                "IndexSpaceBase can only be 1, 2, or 3 Dimensional.");
 
 public:
   /// Constructs a one-dimensional instance and assigns the corresponding data
   /// to Dim0 value. Available only if Dimensions = 1.
   template <int N = Dimensions, std::enable_if_t<N == 1, bool> = true>
-  RawArray(size_t Dim0 = 0) : MArray{Dim0} {}
+  IndexSpaceBase(size_t Dim0 = 0) : MArray{Dim0} {}
 
   /// Constructs a two-dimensional instance and assigns the corresponding data.
   /// Available only if Dimensions = 2.
   template <int N = Dimensions, std::enable_if_t<N == 2, bool> = true>
-  RawArray(size_t Dim0, size_t Dim1) : MArray{Dim0, Dim1} {}
+  IndexSpaceBase(size_t Dim0, size_t Dim1) : MArray{Dim0, Dim1} {}
 
   /// Constructs a two-dimensional instance with the zero-initialized
   /// corresponding data. Available only if Dimensions = 2.
   template <int N = Dimensions, std::enable_if_t<N == 2, bool> = true>
-  RawArray() : RawArray(0, 0) {}
+  IndexSpaceBase() : IndexSpaceBase(0, 0) {}
 
   /// Constructs a three-dimensional instance and assigns the corresponding
   /// data. Available only if Dimensions = 3.
   template <int N = Dimensions, std::enable_if_t<N == 3, bool> = true>
-  RawArray(size_t Dim0, size_t Dim1, size_t Dim2) : MArray{Dim0, Dim1, Dim2} {}
+  IndexSpaceBase(size_t Dim0, size_t Dim1, size_t Dim2)
+      : MArray{Dim0, Dim1, Dim2} {}
 
   /// Constructs a three-dimensional instance with the zero-initialized
   /// corresponding data. Available only if Dimensions = 3.
   template <int N = Dimensions, std::enable_if_t<N == 3, bool> = true>
-  RawArray() : RawArray(0, 0, 0) {}
+  IndexSpaceBase() : IndexSpaceBase(0, 0, 0) {}
 
   /// Returns the value for the specified dimension.
   /// Results in undefined behavior if dimension is not in the range [0,
@@ -81,14 +85,16 @@ template <int Dimensions = 1> class RawArray {
     return MArray[Dimension];
   }
 
-  RawArray(const RawArray<Dimensions> &rhs) = default;
-  RawArray(RawArray<Dimensions> &&rhs) = default;
-  RawArray<Dimensions> &operator=(const RawArray<Dimensions> &rhs) = default;
-  RawArray<Dimensions> &operator=(RawArray<Dimensions> &&rhs) = default;
-  ~RawArray() = default;
+  IndexSpaceBase(const IndexSpaceBase<Derived, Dimensions> &rhs) = default;
+  IndexSpaceBase(IndexSpaceBase<Derived, Dimensions> &&rhs) = default;
+  IndexSpaceBase<Derived, Dimensions> &
+  operator=(const IndexSpaceBase<Derived, Dimensions> &rhs) = default;
+  IndexSpaceBase<Derived, Dimensions> &
+  operator=(IndexSpaceBase<Derived, Dimensions> &&rhs) = default;
+  ~IndexSpaceBase() = default;
 
-  friend bool operator==(const RawArray<Dimensions> &lhs,
-                         const RawArray<Dimensions> &rhs) {
+  friend bool operator==(const IndexSpaceBase<Derived, Dimensions> &lhs,
+                         const IndexSpaceBase<Derived, Dimensions> &rhs) {
     for (int i = 0; i < Dimensions; ++i) {
       if (lhs.MArray[i] != rhs.MArray[i]) {
         return false;
@@ -97,14 +103,127 @@ template <int Dimensions = 1> class RawArray {
     return true;
   }
 
-  friend bool operator!=(const RawArray<Dimensions> &lhs,
-                         const RawArray<Dimensions> &rhs) {
+  friend bool operator!=(const IndexSpaceBase<Derived, Dimensions> &lhs,
+                         const IndexSpaceBase<Derived, Dimensions> &rhs) {
     return !(lhs == rhs);
   }
 
+#define _LIBSYCL_GEN_OPT(op)                                                   \
+  friend Derived operator op(const Derived &lhs,                               \
+                             const Derived &rhs) noexcept {                    \
+    Derived result;                                                            \
+    for (int i = 0; i < Dimensions; ++i) {                                     \
+      result.MArray[i] = lhs.MArray[i] op rhs.MArray[i];                       \
+    }                                                                          \
+    return result;                                                             \
+  }                                                                            \
+                                                                               \
+  template <typename T>                                                        \
+  friend IntegralType<T, Derived> operator op(const Derived &lhs,              \
+                                              const T &rhs) noexcept {         \
+    Derived result;                                                            \
+    for (int i = 0; i < Dimensions; ++i) {                                     \
+      result.MArray[i] = lhs.MArray[i] op rhs;                                 \
+    }                                                                          \
+    return result;                                                             \
+  }                                                                            \
+                                                                               \
+  template <typename T>                                                        \
+  friend IntegralType<T, Derived> operator op(const T &lhs,                    \
+                                              const Derived &rhs) noexcept {   \
+    Derived result;                                                            \
+    for (int i = 0; i < Dimensions; ++i) {                                     \
+      result.MArray[i] = lhs op rhs.MArray[i];                                 \
+    }                                                                          \
+    return result;                                                             \
+  }
+
+  _LIBSYCL_GEN_OPT(+)
+  _LIBSYCL_GEN_OPT(-)
+  _LIBSYCL_GEN_OPT(*)
+  _LIBSYCL_GEN_OPT(/)
+  _LIBSYCL_GEN_OPT(%)
+  _LIBSYCL_GEN_OPT(<<)
+  _LIBSYCL_GEN_OPT(>>)
+  _LIBSYCL_GEN_OPT(&)
+  _LIBSYCL_GEN_OPT(|)
+  _LIBSYCL_GEN_OPT(^)
+  _LIBSYCL_GEN_OPT(&&)
+  _LIBSYCL_GEN_OPT(||)
+  _LIBSYCL_GEN_OPT(<)
+  _LIBSYCL_GEN_OPT(>)
+  _LIBSYCL_GEN_OPT(<=)
+  _LIBSYCL_GEN_OPT(>=)
+
+#undef _LIBSYCL_GEN_OPT
+
+#define _LIBSYCL_GEN_OPT(op)                                                   \
+  friend Derived &operator op(Derived &lhs, const Derived &rhs) noexcept {     \
+    for (int i = 0; i < Dimensions; ++i) {                                     \
+      lhs.MArray[i] op rhs[i];                                                 \
+    }                                                                          \
+    return lhs;                                                                \
+  }                                                                            \
+  template <typename T>                                                        \
+  friend IntegralType<T, Derived> &operator op(Derived &lhs,                   \
+                                               const T &rhs) noexcept {        \
+    for (int i = 0; i < Dimensions; ++i) {                                     \
+      lhs.MArray[i] op rhs;                                                    \
+    }                                                                          \
+    return lhs;                                                                \
+  }
+
+  _LIBSYCL_GEN_OPT(+=)
+  _LIBSYCL_GEN_OPT(-=)
+  _LIBSYCL_GEN_OPT(*=)
+  _LIBSYCL_GEN_OPT(/=)
+  _LIBSYCL_GEN_OPT(%=)
+  _LIBSYCL_GEN_OPT(<<=)
+  _LIBSYCL_GEN_OPT(>>=)
+  _LIBSYCL_GEN_OPT(&=)
+  _LIBSYCL_GEN_OPT(|=)
+  _LIBSYCL_GEN_OPT(^=)
+
+#undef _LIBSYCL_GEN_OPT
+
+#define _LIBSYCL_GEN_OPT(op)                                                   \
+  friend Derived operator op(const Derived &rhs) noexcept {                    \
+    Derived result;                                                            \
+    for (int i = 0; i < Dimensions; ++i) {                                     \
+      result.MArray[i] = (op rhs.MArray[i]);                                   \
+    }                                                                          \
+    return result;                                                             \
+  }
+
+  _LIBSYCL_GEN_OPT(+)
+  _LIBSYCL_GEN_OPT(-)
+
+#undef _LIBSYCL_GEN_OPT
+
+#define _LIBSYCL_GEN_OPT(op)                                                   \
+  friend Derived &operator op(Derived &rhs) noexcept {                         \
+    for (int i = 0; i < Dimensions; ++i) {                                     \
+      op rhs.MArray[i];                                                        \
+    }                                                                          \
+    return rhs;                                                                \
+  }                                                                            \
+  friend Derived operator op(Derived &lhs, int) noexcept {                     \
+    Derived oldLhs(lhs);                                                       \
+    for (int i = 0; i < Dimensions; ++i) {                                     \
+      op lhs.MArray[i];                                                        \
+    }                                                                          \
+    return oldLhs;                                                             \
+  }
+
+  _LIBSYCL_GEN_OPT(++)
+  _LIBSYCL_GEN_OPT(--)
+
+#undef _LIBSYCL_GEN_OPT
+
 protected:
   size_t MArray[Dimensions];
 };
+
 } // namespace detail
 
 /// SYCL 2020 4.9.1.1. range class.
@@ -112,10 +231,10 @@ template <int Dimensions = 1> class RawArray {
 /// domain of either a single work-group in a parallel dispatch, or the overall
 /// Dimensions of the dispatch.
 template <int Dimensions = 1>
-class range : public detail::RawArray<Dimensions> {
+class range : public detail::IndexSpaceBase<range<Dimensions>, Dimensions> {
   static_assert(Dimensions >= 1 && Dimensions <= 3,
                 "range can only be 1-, 2-, or 3-dimensional.");
-  using Base = detail::RawArray<Dimensions>;
+  using Base = detail::IndexSpaceBase<range<Dimensions>, Dimensions>;
 
 public:
   static constexpr int dimensions = Dimensions;
@@ -142,7 +261,7 @@ class range : public detail::RawArray<Dimensions> {
       : Base(dim0, dim1, dim2) {}
 
   /*
-  Declared and implemented in detail::RawArray:
+  Declared and implemented in detail::IndexSpaceBase:
       std::size_t get(int dimension) const noexcept;
       std::size_t& operator[](int dimension) noexcept;
       std::size_t operator[](int dimension) const noexcept;
@@ -156,8 +275,6 @@ class range : public detail::RawArray<Dimensions> {
     }
     return size;
   }
-
-  // TODO: operators to be added
 };
 
 /// c++ deduction guides.
@@ -173,10 +290,11 @@ template <int Dimensions = 1, bool WithOffset = true> class item;
 /// id<int Dimensions> is a vector of Dimensions that is used to represent an id
 /// into a global or local range. It can be used as an index in an accessor of
 /// the same rank.
-template <int Dimensions = 1> class id : public detail::RawArray<Dimensions> {
+template <int Dimensions = 1>
+class id : public detail::IndexSpaceBase<id<Dimensions>, Dimensions> {
   static_assert(Dimensions >= 1 && Dimensions <= 3,
                 "id can only be 1-, 2-, or 3-dimensional.");
-  using Base = detail::RawArray<Dimensions>;
+  using Base = detail::IndexSpaceBase<id<Dimensions>, Dimensions>;
 
   // Helper class for conversion operator. Void type is not suitable. User
   // cannot even try to get address of the operator PrivateTag(). User
@@ -230,39 +348,67 @@ template <int Dimensions = 1> class id : public detail::RawArray<Dimensions> {
 
   /// Constructs an id from item.get_id().
   /// Only valid when the template parameter Dimensions is equal to 1.
-  template <int N = Dimensions, std::enable_if_t<N == 1, bool> = true>
-  id(const item<Dimensions> &item) noexcept : Base(item.get_id(0)) {}
+  template <int N = Dimensions, bool WithOffset = true,
+            std::enable_if_t<N == 1, bool> = true>
+  id(const item<Dimensions, WithOffset> &item) noexcept
+      : Base(item.get_id(0)) {}
 
   /// Constructs an id from item.get_id().
   /// Only valid when the template parameter Dimensions is equal to 2.
-  template <int N = Dimensions, std::enable_if_t<N == 2, bool> = true>
-  id(const item<Dimensions> &item) noexcept
+  template <int N = Dimensions, bool WithOffset = true,
+            std::enable_if_t<N == 2, bool> = true>
+  id(const item<Dimensions, WithOffset> &item) noexcept
       : Base(item.get_id(0), item.get_id(1)) {}
 
   /// Constructs an id from item.get_id().
   /// Only valid when the template parameter Dimensions is equal to 3.
-  template <int N = Dimensions, std::enable_if_t<N == 3, bool> = true>
-  id(const item<Dimensions> &item) noexcept
+  template <int N = Dimensions, bool WithOffset = true,
+            std::enable_if_t<N == 3, bool> = true>
+  id(const item<Dimensions, WithOffset> &item) noexcept
       : Base(item.get_id(0), item.get_id(1), item.get_id(2)) {}
 
   /*
-    Declared and implemented in detail::RawArray:
+    Declared and implemented in detail::IndexSpaceBase:
         std::size_t get(int dimension) const noexcept;
         std::size_t& operator[](int dimension) noexcept;
         std::size_t operator[](int dimension) const noexcept;
     */
 
   // Template operator is not allowed because it disables further type
-  //   conversion. For example, the next code will not work in case of template
-  //   conversion:
-  //   int a = id<1>(value);
+  // conversion. For example, the next code will not work in case of template
+  // conversion: int a = id<1>(value);
   /// Returns the same value as get(0).
   ///  Available only when: Dimensions == 1.
   operator EnableIfT<(Dimensions == 1), std::size_t>() const noexcept {
     return Base::get(0);
   }
 
-  // TODO: operators to be added
+// These operators are not a part of SYCL 2020 spec but are needed to avoid
+// ambiguity in case of implicit conversion  id<1> vs size_t. Template operators
+// take precedence over type conversion. In the case of non-template operators,
+// ambiguity appears: "id op size_t" may refer "size_t op size_t" or "id op
+// size_t". In the case of template operators it will be "id op size_t".
+#define _LIBSYCL_GEN_OPT(op)                                                   \
+  template <typename T, int N = Dimensions,                                    \
+            std::enable_if_t<N == 1, bool> = true>                             \
+  detail::IntegralType<T, bool> operator op(const T &rhs) const noexcept {     \
+    if (this->MArray[0] != rhs)                                                \
+      return false op true;                                                    \
+    return true op true;                                                       \
+  }                                                                            \
+  template <typename T, int N = Dimensions,                                    \
+            std::enable_if_t<N == 1, bool> = true>                             \
+  friend detail::IntegralType<T, bool> operator op(                            \
+      const T &lhs, const id<dimensions> &rhs) noexcept {                      \
+    if (lhs != rhs.MArray[0])                                                  \
+      return false op true;                                                    \
+    return true op true;                                                       \
+  }
+
+  _LIBSYCL_GEN_OPT(==)
+  _LIBSYCL_GEN_OPT(!=)
+
+#undef _LIBSYCL_GEN_OPT
 };
 
 /// c++ deduction guides.
diff --git a/libsycl/include/sycl/__impl/queue.hpp b/libsycl/include/sycl/__impl/queue.hpp
index 87f1a4d7e6c14..d05592e5fa3fd 100644
--- a/libsycl/include/sycl/__impl/queue.hpp
+++ b/libsycl/include/sycl/__impl/queue.hpp
@@ -32,6 +32,7 @@ _LIBSYCL_BEGIN_NAMESPACE_SYCL
 class context;
 
 namespace detail {
+class MockQueue;
 class QueueImpl;
 
 template <typename, typename T> struct CheckFunctionCallOperator {
@@ -430,6 +431,7 @@ class _LIBSYCL_EXPORT queue {
   std::shared_ptr<detail::QueueImpl> impl;
 
   friend sycl::detail::ImplUtils;
+  friend sycl::detail::MockQueue;
 }; // class queue
 
 _LIBSYCL_END_NAMESPACE_SYCL
diff --git a/libsycl/test/basic/index_space_classes.cpp b/libsycl/test/basic/index_space_classes.cpp
new file mode 100644
index 0000000000000..1afb70e634303
--- /dev/null
+++ b/libsycl/test/basic/index_space_classes.cpp
@@ -0,0 +1,284 @@
+// REQUIRES: any-device
+// RUN: %clangxx -fsycl %s -o %t.out
+// RUN: %t.out
+//
+// Unified test for sycl::range and sycl::id covering all operators defined in
+// theirs base class, plus class-specific behaviour for each type.
+
+#include <cassert>
+#include <sycl/sycl.hpp>
+#include <type_traits>
+
+using sycl::detail::Builder;
+
+// Helper to create values for any dimension from three components.
+// Only uses the required number of values based on dimension.
+template <template <int> class T, int Dim>
+T<Dim> makeValue(std::size_t a, std::size_t b = 0, std::size_t c = 0) {
+  if constexpr (Dim == 1)
+    return T<1>(a);
+  else if constexpr (Dim == 2)
+    return T<2>(a, b);
+  else
+    return T<3>(a, b, c);
+}
+
+// Tests binary and compound operators for a specific dimension.
+template <template <int> class T, int Dim> void testBinaryOpsForDim() {
+  const T<Dim> P = makeValue<T, Dim>(8, 9, 10);
+  const T<Dim> Q = makeValue<T, Dim>(2, 3, 5);
+
+  // Binary operators: object op object.
+  assert((P + Q) == (makeValue<T, Dim>(10, 12, 15)));
+  assert((P - Q) == (makeValue<T, Dim>(6, 6, 5)));
+  assert((P * Q) == (makeValue<T, Dim>(16, 27, 50)));
+  assert((P / Q) == (makeValue<T, Dim>(4, 3, 2)));
+  assert((P % Q) == (makeValue<T, Dim>(0, 0, 0)));
+  assert((P << Q) == (makeValue<T, Dim>(32, 72, 320)));
+  assert((P >> Q) == (makeValue<T, Dim>(2, 1, 0)));
+  assert((P & Q) == (makeValue<T, Dim>(0, 1, 0)));
+  assert((P | Q) == (makeValue<T, Dim>(10, 11, 15)));
+  assert((P ^ Q) == (makeValue<T, Dim>(10, 10, 15)));
+  assert((P && Q) == (makeValue<T, Dim>(1, 1, 1)));
+  assert((P || Q) == (makeValue<T, Dim>(1, 1, 1)));
+  assert((P < Q) == (makeValue<T, Dim>(0, 0, 0)));
+  assert((P > Q) == (makeValue<T, Dim>(1, 1, 1)));
+  assert((P <= Q) == (makeValue<T, Dim>(0, 0, 0)));
+  assert((P >= Q) == (makeValue<T, Dim>(1, 1, 1)));
+
+  // Binary operators: object op scalar.
+  assert((P + 1) == (makeValue<T, Dim>(9, 10, 11)));
+  assert((P - 1) == (makeValue<T, Dim>(7, 8, 9)));
+  assert((P * 2) == (makeValue<T, Dim>(16, 18, 20)));
+  assert((P / 2) == (makeValue<T, Dim>(4, 4, 5)));
+  assert((P % 4) == (makeValue<T, Dim>(0, 1, 2)));
+  assert((P << 1) == (makeValue<T, Dim>(16, 18, 20)));
+  assert((P >> 1) == (makeValue<T, Dim>(4, 4, 5)));
+  assert((P & 6) == (makeValue<T, Dim>(0, 0, 2)));
+  assert((P | 1) == (makeValue<T, Dim>(9, 9, 11)));
+  assert((P ^ 3) == (makeValue<T, Dim>(11, 10, 9)));
+  assert((P && 0) == (makeValue<T, Dim>(0, 0, 0)));
+  assert((P || 0) == (makeValue<T, Dim>(1, 1, 1)));
+  assert((P < 9) == (makeValue<T, Dim>(1, 0, 0)));
+  assert((P > 9) == (makeValue<T, Dim>(0, 0, 1)));
+  assert((P <= 9) == (makeValue<T, Dim>(1, 1, 0)));
+  assert((P >= 9) == (makeValue<T, Dim>(0, 1, 1)));
+
+  // Binary operators: scalar op object.
+  assert((1 + P) == (makeValue<T, Dim>(9, 10, 11)));
+  assert((20 - Q) == (makeValue<T, Dim>(18, 17, 15)));
+  assert((2 * P) == (makeValue<T, Dim>(16, 18, 20)));
+  assert((20 / Q) == (makeValue<T, Dim>(10, 6, 4)));
+  assert((33 % Q) == (makeValue<T, Dim>(1, 0, 3)));
+  assert((1 << Q) == (makeValue<T, Dim>(4, 8, 32)));
+  assert((256 >> Q) == (makeValue<T, Dim>(64, 32, 8)));
+  assert((15 & Q) == (makeValue<T, Dim>(2, 3, 5)));
+  assert((1 | Q) == (makeValue<T, Dim>(3, 3, 5)));
+  assert((3 ^ Q) == (makeValue<T, Dim>(1, 0, 6)));
+  assert((1 && Q) == (makeValue<T, Dim>(1, 1, 1)));
+  assert((0 || Q) == (makeValue<T, Dim>(1, 1, 1)));
+  assert((9 < P) == (makeValue<T, Dim>(0, 0, 1)));
+  assert((9 > P) == (makeValue<T, Dim>(1, 0, 0)));
+  assert((9 <= P) == (makeValue<T, Dim>(0, 1, 1)));
+  assert((9 >= P) == (makeValue<T, Dim>(1, 1, 0)));
+
+  // Compound assignment: object op= object.
+  T<Dim> E = makeValue<T, Dim>(8, 9, 10);
+  E += Q;
+  assert(E == (makeValue<T, Dim>(10, 12, 15)));
+  E -= Q;
+  assert(E == (makeValue<T, Dim>(8, 9, 10)));
+  E *= Q;
+  assert(E == (makeValue<T, Dim>(16, 27, 50)));
+  E /= Q;
+  assert(E == (makeValue<T, Dim>(8, 9, 10)));
+  E %= Q;
+  assert(E == (makeValue<T, Dim>(0, 0, 0)));
+
+  // Compound assignment: object op= scalar.
+  T<Dim> F = makeValue<T, Dim>(8, 9, 10);
+  F += 2;
+  assert(F == (makeValue<T, Dim>(10, 11, 12)));
+  F -= 2;
+  assert(F == (makeValue<T, Dim>(8, 9, 10)));
+  F *= 2;
+  assert(F == (makeValue<T, Dim>(16, 18, 20)));
+  F /= 2;
+  assert(F == (makeValue<T, Dim>(8, 9, 10)));
+  F %= 6;
+  assert(F == (makeValue<T, Dim>(2, 3, 4)));
+  F <<= 1;
+  assert(F == (makeValue<T, Dim>(4, 6, 8)));
+  F >>= 1;
+  assert(F == (makeValue<T, Dim>(2, 3, 4)));
+  F &= 6;
+  assert(F == (makeValue<T, Dim>(2, 2, 4)));
+  F |= 1;
+  assert(F == (makeValue<T, Dim>(3, 3, 5)));
+  F ^= 2;
+  assert(F == (makeValue<T, Dim>(1, 1, 7)));
+}
+
+template <template <int> class T> void testCommon() {
+  // Default construction zero-initialises every dimension.
+  T<1> Def1;
+  T<2> Def2;
+  T<3> Def3;
+  assert(Def1[0] == 0);
+  assert(Def2[0] == 0 && Def2[1] == 0);
+  assert(Def3[0] == 0 && Def3[1] == 0 && Def3[2] == 0);
+
+  // Dimensional constructors and element access via get() and const [].
+  T<1> A1(5);
+  T<2> A2(3, 7);
+  T<3> A3(8, 9, 10);
+  assert(A1.get(0) == 5 && A1[0] == 5);
+  assert(A2.get(0) == 3 && A2.get(1) == 7);
+  assert(A3.get(0) == 8 && A3.get(1) == 9 && A3.get(2) == 10);
+
+  // dimensions static constexpr.
+  static_assert(T<1>::dimensions == 1);
+  static_assert(T<2>::dimensions == 2);
+  static_assert(T<3>::dimensions == 3);
+
+  // Non-const operator[] write.
+  T<2> Mutable(1, 2);
+  Mutable[0] = 10;
+  Mutable[1] = 20;
+  assert(Mutable[0] == 10 && Mutable[1] == 20);
+
+  // Copy constructor and copy assignment.
+  T<3> CopyCtor(A3);
+  assert(CopyCtor == A3);
+  T<3> CopyAssign(1, 1, 1);
+  CopyAssign = A3;
+  assert(CopyAssign == A3);
+
+  // Move constructor and move assignment.
+  T<3> MoveSrc1(A3);
+  T<3> MoveCtor(std::move(MoveSrc1));
+  assert(MoveCtor == A3);
+  T<3> MoveSrc2(A3);
+  T<3> MoveAssign(1, 1, 1);
+  MoveAssign = std::move(MoveSrc2);
+  assert(MoveAssign == A3);
+
+  // Equality and inequality between same-type objects.
+  assert((makeValue<T, 3>(8, 9, 10)) == (makeValue<T, 3>(8, 9, 10)));
+  assert((makeValue<T, 3>(8, 9, 10)) != (makeValue<T, 3>(8, 9, 11)));
+
+  // Test binary and compound operators for all dimensions.
+  testBinaryOpsForDim<T, 1>();
+  testBinaryOpsForDim<T, 2>();
+  testBinaryOpsForDim<T, 3>();
+
+  // Unary + and -.
+  T<3> U(8, 9, 10);
+  assert((+U) == (makeValue<T, 3>(8, 9, 10)));
+  assert((-U) == (makeValue<T, 3>(static_cast<std::size_t>(-8),
+                                  static_cast<std::size_t>(-9),
+                                  static_cast<std::size_t>(-10))));
+
+  // Pre- and post-increment / decrement.
+  T<3> Inc(8, 9, 10);
+  assert((++Inc) == (makeValue<T, 3>(9, 10, 11)));
+  assert((Inc++) == (makeValue<T, 3>(9, 10, 11)));
+  assert(Inc == (makeValue<T, 3>(10, 11, 12)));
+  assert((--Inc) == (makeValue<T, 3>(9, 10, 11)));
+  assert((Inc--) == (makeValue<T, 3>(9, 10, 11)));
+  assert(Inc == (makeValue<T, 3>(8, 9, 10)));
+}
+
+void testRange() {
+  testCommon<sycl::range>();
+
+  assert(sycl::range<1>(7).size() == 7);
+  assert(sycl::range<2>(4, 5).size() == 20);
+  assert(sycl::range<3>(2, 3, 4).size() == 24);
+  assert(sycl::range<3>(0, 5, 3).size() == 0);
+
+#ifdef __cpp_deduction_guides
+  auto R1 = sycl::range(7);
+  auto R2 = sycl::range(4, 5);
+  auto R3 = sycl::range(2, 3, 4);
+  static_assert(std::is_same_v<decltype(R1), sycl::range<1>>);
+  static_assert(std::is_same_v<decltype(R2), sycl::range<2>>);
+  static_assert(std::is_same_v<decltype(R3), sycl::range<3>>);
+  assert(R1 == sycl::range<1>(7));
+  assert(R2 == sycl::range<2>(4, 5));
+  assert(R3 == sycl::range<3>(2, 3, 4));
+#endif
+}
+
+void testId() {
+  testCommon<sycl::id>();
+
+  // Construction from range.
+  assert(sycl::id<1>(sycl::range<1>(2)) == sycl::id<1>(2));
+  assert(sycl::id<2>(sycl::range<2>(4, 8)) == sycl::id<2>(4, 8));
+  assert(sycl::id<3>(sycl::range<3>(16, 32, 64)) == sycl::id<3>(16, 32, 64));
+
+  // Construction from item (WithOffset=true) preserves the computed id
+  // coordinates.
+  sycl::item<1, true> Item1 = Builder::createItem<1, true>({4}, {2}, {1});
+  sycl::item<2, true> Item2 =
+      Builder::createItem<2, true>({8, 16}, {4, 8}, {1, 1});
+  sycl::item<3, true> Item3 =
+      Builder::createItem<3, true>({32, 64, 128}, {16, 32, 64}, {1, 1, 1});
+  assert(sycl::id<1>(Item1) == sycl::id<1>(2));
+  assert(sycl::id<2>(Item2) == sycl::id<2>(4, 8));
+  assert(sycl::id<3>(Item3) == sycl::id<3>(16, 32, 64));
+
+  // Construction from item (WithOffset=false).
+  sycl::item<1, false> Item1NoOffset = Builder::createItem<1, false>({4}, {2});
+  sycl::item<2, false> Item2NoOffset =
+      Builder::createItem<2, false>({8, 16}, {4, 8});
+  sycl::item<3, false> Item3NoOffset =
+      Builder::createItem<3, false>({32, 64, 128}, {16, 32, 64});
+  assert(sycl::id<1>(Item1NoOffset) == sycl::id<1>(2));
+  assert(sycl::id<2>(Item2NoOffset) == sycl::id<2>(4, 8));
+  assert(sycl::id<3>(Item3NoOffset) == sycl::id<3>(16, 32, 64));
+
+  // Copy-initialization from item must also select id(item) constructor.
+  sycl::id<1> IdFromItem1NoOffset = Item1NoOffset;
+  sycl::id<2> IdFromItem2NoOffset = Item2NoOffset;
+  sycl::id<3> IdFromItem3NoOffset = Item3NoOffset;
+  assert(IdFromItem1NoOffset == sycl::id<1>(2));
+  assert(IdFromItem2NoOffset == sycl::id<2>(4, 8));
+  assert(IdFromItem3NoOffset == sycl::id<3>(16, 32, 64));
+
+#ifdef __cpp_deduction_guides
+  auto I1 = sycl::id(2);
+  auto I2 = sycl::id(4, 8);
+  auto I3 = sycl::id(16, 32, 64);
+  static_assert(std::is_same_v<decltype(I1), sycl::id<1>>);
+  static_assert(std::is_same_v<decltype(I2), sycl::id<2>>);
+  static_assert(std::is_same_v<decltype(I3), sycl::id<3>>);
+  assert(I1 == sycl::id<1>(2));
+  assert(I2 == sycl::id<2>(4, 8));
+  assert(I3 == sycl::id<3>(16, 32, 64));
+#endif
+
+  // Implicit conversion to size_t (id<1> only).
+  sycl::id<1> OneDim(16);
+  std::size_t S = OneDim;
+  int I = OneDim;
+  assert(S == 16 && I == 16);
+
+  // Scalar equality/inequality operators (id<1> only).
+  const sycl::id<1> ConstId(10);
+  assert(ConstId == 10);
+  assert(10 == ConstId);
+  assert(ConstId != 19);
+  assert(19 != ConstId);
+  assert(ConstId == 10u);
+  assert(10u == ConstId);
+  assert(ConstId != static_cast<short>(9));
+  assert(static_cast<short>(9) != ConstId);
+}
+
+int main() {
+  testRange();
+  testId();
+  return 0;
+}
diff --git a/libsycl/test/basic/index_space_classes_negative.cpp b/libsycl/test/basic/index_space_classes_negative.cpp
new file mode 100644
index 0000000000000..4535e2e634269
--- /dev/null
+++ b/libsycl/test/basic/index_space_classes_negative.cpp
@@ -0,0 +1,16 @@
+// RUN: %clangxx -fsycl %s -fsyntax-only -Xclang -verify \
+// RUN: -Xclang -verify-ignore-unexpected=note
+
+#include <sycl/sycl.hpp>
+
+void testInvalidDimensionGatedAPI() {
+  sycl::id<2> Id2(1, 2);
+  std::size_t ScalarFromId2 = Id2; // expected-error {{no viable conversion}}
+  (void)ScalarFromId2;
+
+  sycl::id<2> BadId2(7);          // expected-error {{no matching constructor}}
+  sycl::range<3> BadRange3(1, 2); // expected-error {{no matching constructor}}
+
+  (void)BadId2;
+  (void)BadRange3;
+}
diff --git a/libsycl/unittests/mock/helpers.cpp b/libsycl/unittests/mock/helpers.cpp
index abc68fd2a29d3..6bc89e1b5713f 100644
--- a/libsycl/unittests/mock/helpers.cpp
+++ b/libsycl/unittests/mock/helpers.cpp
@@ -254,13 +254,37 @@ void mock::MockLiboffload::initDefault() {
         return OL_SUCCESS;
       });
 
+  ON_CALL(*this, olLaunchKernel)
+      .WillByDefault([this](ol_queue_handle_t Queue, ol_device_handle_t Device,
+                            ol_symbol_handle_t Kernel,
+                            const ol_kernel_launch_size_args_t *LaunchSizeArgs,
+                            const ol_kernel_launch_prop_t *Properties,
+                            size_t NumArgs, void **ArgPtrs,
+                            const size_t *ArgSizes) -> ol_result_t {
+        if (!Device || !Kernel || !Queue)
+          return makeEmptyStrError(OL_ERRC_INVALID_NULL_HANDLE);
+        if (!LaunchSizeArgs)
+          return makeEmptyStrError(OL_ERRC_INVALID_NULL_POINTER);
+        std::ignore = Properties;
+        if ((ArgPtrs == nullptr) != (ArgSizes == nullptr))
+          return makeEmptyStrError(OL_ERRC_INVALID_ARGUMENT);
+        if (NumArgs > 0 && (!ArgPtrs || !ArgSizes))
+          return makeEmptyStrError(OL_ERRC_INVALID_ARGUMENT);
+        for (size_t I = 0; I < NumArgs; ++I) {
+          if (ArgSizes[I] > 0 && !ArgPtrs[I])
+            return makeEmptyStrError(OL_ERRC_INVALID_ARGUMENT);
+        }
+        return OL_SUCCESS;
+      });
+
   ON_CALL(*this, olCreateEvent)
-      .WillByDefault([this](ol_queue_handle_t Queue,
+      .WillByDefault([this](ol_queue_handle_t Queue, ol_event_flags_t Flags,
                             ol_event_handle_t *Event) -> ol_result_t {
         if (!Queue)
           return makeEmptyStrError(OL_ERRC_INVALID_NULL_HANDLE);
         if (!Event)
           return makeEmptyStrError(OL_ERRC_INVALID_NULL_POINTER);
+        std::ignore = Flags;
         *Event = mock::createDummyHandleWithData<ol_event_handle_t>(
             reinterpret_cast<unsigned char *>(&Queue), sizeof(Queue));
         return OL_SUCCESS;
diff --git a/libsycl/unittests/mock/helpers.hpp b/libsycl/unittests/mock/helpers.hpp
index 5de2ac0098f13..793c687b7276b 100644
--- a/libsycl/unittests/mock/helpers.hpp
+++ b/libsycl/unittests/mock/helpers.hpp
@@ -108,7 +108,14 @@ class MockLiboffload {
               (ol_queue_handle_t Queue, ol_event_handle_t *Events,
                size_t NumEvents));
   MOCK_METHOD(ol_result_t, olCreateEvent,
-              (ol_queue_handle_t Queue, ol_event_handle_t *Event));
+              (ol_queue_handle_t Queue, ol_event_flags_t Flags,
+               ol_event_handle_t *Event));
+  MOCK_METHOD(ol_result_t, olLaunchKernel,
+              (ol_queue_handle_t Queue, ol_device_handle_t Device,
+               ol_symbol_handle_t Kernel,
+               const ol_kernel_launch_size_args_t *LaunchSizeArgs,
+               const ol_kernel_launch_prop_t *Properties, size_t NumArgs,
+               void **ArgPtrs, const size_t *ArgSizes));
 
   ol_result_t makeEmptyStrError(ol_errc_t Code) {
     auto [Iterator, Flag] =
diff --git a/libsycl/unittests/mock/mock.cpp b/libsycl/unittests/mock/mock.cpp
index 4580bd8c6d602..a4e66548c7f40 100644
--- a/libsycl/unittests/mock/mock.cpp
+++ b/libsycl/unittests/mock/mock.cpp
@@ -84,8 +84,20 @@ ol_result_t olWaitEvents(ol_queue_handle_t Queue, ol_event_handle_t *Events,
   return mock::getMockLiboffload().olWaitEvents(Queue, Events, NumEvents);
 }
 
-ol_result_t olCreateEvent(ol_queue_handle_t Queue, ol_event_handle_t *Event) {
-  return mock::getMockLiboffload().olCreateEvent(Queue, Event);
+ol_result_t olLaunchKernel(ol_queue_handle_t Queue, ol_device_handle_t Device,
+                           ol_symbol_handle_t Kernel,
+                           const ol_kernel_launch_size_args_t *LaunchSizeArgs,
+                           const ol_kernel_launch_prop_t *Properties,
+                           size_t NumArgs, void **ArgPtrs,
+                           const size_t *ArgSizes) {
+  return mock::getMockLiboffload().olLaunchKernel(Queue, Device, Kernel,
+                                                  LaunchSizeArgs, Properties,
+                                                  NumArgs, ArgPtrs, ArgSizes);
+}
+
+ol_result_t olCreateEvent(ol_queue_handle_t Queue, ol_event_flags_t Flags,
+                          ol_event_handle_t *Event) {
+  return mock::getMockLiboffload().olCreateEvent(Queue, Flags, Event);
 }
 
 ol_result_t olDestroyEvent(ol_event_handle_t Event) {
diff --git a/libsycl/unittests/queue/CMakeLists.txt b/libsycl/unittests/queue/CMakeLists.txt
index f03220ab10dd1..2c66d3f110451 100644
--- a/libsycl/unittests/queue/CMakeLists.txt
+++ b/libsycl/unittests/queue/CMakeLists.txt
@@ -1,3 +1,4 @@
 add_sycl_unittest(QueueTests
     queue.cpp
+    sycl_kernel_launch.cpp
 )
diff --git a/libsycl/unittests/queue/sycl_kernel_launch.cpp b/libsycl/unittests/queue/sycl_kernel_launch.cpp
new file mode 100644
index 0000000000000..653dc7306d623
--- /dev/null
+++ b/libsycl/unittests/queue/sycl_kernel_launch.cpp
@@ -0,0 +1,98 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <common/device_images.hpp>
+#include <mock/helpers.hpp>
+
+#include <detail/program_manager.hpp>
+
+#include <sycl/__impl/queue.hpp>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include <array>
+#include <cstring>
+#include <string>
+
+using namespace sycl;
+using namespace ::testing;
+
+namespace {
+
+class ScopedKernelRegistration {
+public:
+  explicit ScopedKernelRegistration(std::string KernelName)
+      : MKernelName(std::move(KernelName)) {
+    std::array<llvm::StringRef, 1> KernelNames = {MKernelName};
+    MBinary = sycl::unittest::createSYCLDeviceBinary(KernelNames);
+    sycl::detail::ProgramAndKernelManager::getInstance().registerFatBin(
+        MBinary.data(), MBinary.size());
+  }
+
+  ~ScopedKernelRegistration() {
+    sycl::detail::ProgramAndKernelManager::getInstance().unregisterFatBin(
+        MBinary.data(), MBinary.size());
+  }
+
+private:
+  std::string MKernelName;
+  llvm::SmallString<0> MBinary;
+};
+
+} // namespace
+
+class sycl::detail::MockQueue : public sycl::queue {
+public:
+  using sycl::queue::sycl_kernel_launch;
+};
+
+struct KernelData {
+  char Arr[16];
+  int SomeInt;
+  double DoubleArr[8];
+};
+
+TEST(Queue, KernelLaunch) {
+  mock::MockWrapper Mock;
+  ScopedKernelRegistration Registration{"TestKernel"};
+  sycl::detail::MockQueue Q;
+  KernelData Data{};
+  Data.Arr[0] = 'A';
+  Data.Arr[1] = 'I';
+  Data.SomeInt = 42;
+  Data.DoubleArr[0] = 3.14;
+  Data.DoubleArr[7] = -1.25;
+
+  EXPECT_CALL(Mock.get(), olLaunchKernel(_, _, _, _, _, 1, _, _))
+      .WillOnce([&Data](ol_queue_handle_t Queue, ol_device_handle_t Device,
+                        ol_symbol_handle_t Kernel,
+                        const ol_kernel_launch_size_args_t *LaunchSizeArgs,
+                        const ol_kernel_launch_prop_t *Properties,
+                        size_t NumArgs, void **ArgPtrs,
+                        const size_t *ArgSizes) -> ol_result_t {
+        EXPECT_NE(Queue, nullptr);
+        EXPECT_NE(Device, nullptr);
+        EXPECT_NE(Kernel, nullptr);
+        EXPECT_NE(LaunchSizeArgs, nullptr);
+        std::ignore = Properties;
+        EXPECT_EQ(NumArgs, 1u);
+        EXPECT_NE(ArgPtrs, nullptr);
+        EXPECT_NE(ArgSizes, nullptr);
+        EXPECT_EQ(ArgSizes[0], sizeof(KernelData));
+
+        auto PayloadPtr = static_cast<const KernelData *>(ArgPtrs[0]);
+        EXPECT_NE(PayloadPtr, nullptr);
+        EXPECT_EQ(std::memcmp(PayloadPtr, &Data, sizeof(KernelData)), 0);
+        return OL_SUCCESS;
+      });
+  Q.sycl_kernel_launch<class TestKernel>("TestKernel", Data);
+
+  EXPECT_CALL(Mock.get(), olSyncQueue(_)).Times(1);
+  Q.wait();
+}
diff --git a/lldb/examples/python/templates/scripted_process.py b/lldb/examples/python/templates/scripted_process.py
index af21c80e6896d..18777ded6c556 100644
--- a/lldb/examples/python/templates/scripted_process.py
+++ b/lldb/examples/python/templates/scripted_process.py
@@ -532,6 +532,13 @@ def get_register_info(self):
                 elif "hexagon" in self.arch:
                     self.register_info["sets"] = ["General Purpose Registers"]
                     self.register_info["registers"] = HEXAGON_GPR
+                elif "wasm" in self.arch:
+                    # WebAssembly is a stack machine with no general-purpose
+                    # register file, so there is no GPR layout to expose. Leave
+                    # the register info empty rather than raising, which would
+                    # otherwise prevent the scripted frame from constructing.
+                    self.register_info["sets"] = []
+                    self.register_info["registers"] = []
                 else:
                     raise ValueError("Unknown architecture", self.arch)
         return self.register_info
diff --git a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp
index cae579e533523..4cf2d3d35c85e 100644
--- a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp
+++ b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp
@@ -27,51 +27,13 @@
 #include "Plugins/Process/Utility/RegisterFlagsDetector_arm64.h"
 #include "Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h"
 
+#include "llvm/BinaryFormat/ELF.h"
+
 // System includes - They have to be included after framework includes because
 // they define some macros which collide with variable names in other modules
-#include <sys/uio.h>
-// NT_PRSTATUS and NT_FPREGSET definition
-#include <elf.h>
 #include <mutex>
 #include <optional>
-
-#ifndef NT_ARM_SVE
-#define NT_ARM_SVE 0x405 /* ARM Scalable Vector Extension */
-#endif
-
-#ifndef NT_ARM_SSVE
-#define NT_ARM_SSVE                                                            \
-  0x40b /* ARM Scalable Matrix Extension, Streaming SVE mode */
-#endif
-
-#ifndef NT_ARM_ZA
-#define NT_ARM_ZA 0x40c /* ARM Scalable Matrix Extension, Array Storage */
-#endif
-
-#ifndef NT_ARM_ZT
-#define NT_ARM_ZT                                                              \
-  0x40d /* ARM Scalable Matrix Extension 2, lookup table register */
-#endif
-
-#ifndef NT_ARM_PAC_MASK
-#define NT_ARM_PAC_MASK 0x406 /* Pointer authentication code masks */
-#endif
-
-#ifndef NT_ARM_TAGGED_ADDR_CTRL
-#define NT_ARM_TAGGED_ADDR_CTRL 0x409 /* Tagged address control register */
-#endif
-
-#ifndef NT_ARM_FPMR
-#define NT_ARM_FPMR 0x40e /* Floating point mode register */
-#endif
-
-#ifndef NT_ARM_POE
-#define NT_ARM_POE 0x40f /* Permission Overlay registers */
-#endif
-
-#ifndef NT_ARM_GCS
-#define NT_ARM_GCS 0x410 /* Guarded Control Stack control registers */
-#endif
+#include <sys/uio.h>
 
 #ifndef HWCAP_PACA
 #define HWCAP_PACA (1 << 30)
@@ -118,7 +80,7 @@ NativeRegisterContextLinux::CreateHostNativeRegisterContextLinux(
     struct iovec ioVec;
     ioVec.iov_base = &sve_header;
     ioVec.iov_len = sizeof(sve_header);
-    unsigned int regset = NT_ARM_SVE;
+    unsigned int regset = llvm::ELF::NT_ARM_SVE;
 
     Flags opt_regsets;
     if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,
@@ -130,7 +92,7 @@ NativeRegisterContextLinux::CreateHostNativeRegisterContextLinux(
     // We may have the Scalable Matrix Extension (SME) which adds a
     // streaming SVE mode. Systems can have SVE and/or SME.
     ioVec.iov_len = sizeof(sve_header);
-    regset = NT_ARM_SSVE;
+    regset = llvm::ELF::NT_ARM_SSVE;
     if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,
                                           native_thread.GetID(), &regset,
                                           &ioVec, sizeof(sve_header))
@@ -140,7 +102,7 @@ NativeRegisterContextLinux::CreateHostNativeRegisterContextLinux(
     sve::user_za_header za_header;
     ioVec.iov_base = &za_header;
     ioVec.iov_len = sizeof(za_header);
-    regset = NT_ARM_ZA;
+    regset = llvm::ELF::NT_ARM_ZA;
     if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,
                                           native_thread.GetID(), &regset,
                                           &ioVec, sizeof(za_header))
@@ -151,7 +113,7 @@ NativeRegisterContextLinux::CreateHostNativeRegisterContextLinux(
     std::array<uint8_t, 64> zt_reg;
     ioVec.iov_base = zt_reg.data();
     ioVec.iov_len = zt_reg.size();
-    regset = NT_ARM_ZT;
+    regset = llvm::ELF::NT_ARM_ZT;
     if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,
                                           native_thread.GetID(), &regset,
                                           &ioVec, zt_reg.size())
@@ -1190,7 +1152,8 @@ Status NativeRegisterContextLinux_arm64::WriteAllRegisterValues(
 
         // Even though the system does not have SVE, NT_ARM_SVE is used when
         // exiting streaming mode.
-        error = WriteRegisterSet(&ioVec, sve_fpsimd_data.size(), NT_ARM_SVE);
+        error = WriteRegisterSet(&ioVec, sve_fpsimd_data.size(),
+                                 llvm::ELF::NT_ARM_SVE);
 
         // Consume FP register set.
         src += GetFPRSize();
@@ -1338,7 +1301,7 @@ Status NativeRegisterContextLinux_arm64::ReadGPR() {
   ioVec.iov_base = GetGPRBuffer();
   ioVec.iov_len = GetGPRBufferSize();
 
-  error = ReadRegisterSet(&ioVec, GetGPRBufferSize(), NT_PRSTATUS);
+  error = ReadRegisterSet(&ioVec, GetGPRBufferSize(), llvm::ELF::NT_PRSTATUS);
 
   if (error.Success())
     m_gpr_is_valid = true;
@@ -1357,7 +1320,7 @@ Status NativeRegisterContextLinux_arm64::WriteGPR() {
 
   m_gpr_is_valid = false;
 
-  return WriteRegisterSet(&ioVec, GetGPRBufferSize(), NT_PRSTATUS);
+  return WriteRegisterSet(&ioVec, GetGPRBufferSize(), llvm::ELF::NT_PRSTATUS);
 }
 
 Status NativeRegisterContextLinux_arm64::ReadFPR() {
@@ -1370,7 +1333,7 @@ Status NativeRegisterContextLinux_arm64::ReadFPR() {
   ioVec.iov_base = GetFPRBuffer();
   ioVec.iov_len = GetFPRSize();
 
-  error = ReadRegisterSet(&ioVec, GetFPRSize(), NT_FPREGSET);
+  error = ReadRegisterSet(&ioVec, GetFPRSize(), llvm::ELF::NT_FPREGSET);
   if (error.Success())
     m_fpu_is_valid = true;
 
@@ -1391,7 +1354,7 @@ Status NativeRegisterContextLinux_arm64::WriteFPR() {
   m_sve_buffer_is_valid = false;
   m_sve_header_is_valid = false;
 
-  return WriteRegisterSet(&ioVec, GetFPRSize(), NT_FPREGSET);
+  return WriteRegisterSet(&ioVec, GetFPRSize(), llvm::ELF::NT_FPREGSET);
 }
 
 void NativeRegisterContextLinux_arm64::InvalidateAllRegisters() {
@@ -1417,9 +1380,9 @@ unsigned NativeRegisterContextLinux_arm64::GetSVERegSet() {
   switch (m_sve_state) {
   case SVEState::Streaming:
   case SVEState::StreamingFPSIMD:
-    return NT_ARM_SSVE;
+    return llvm::ELF::NT_ARM_SSVE;
   default:
-    return NT_ARM_SVE;
+    return llvm::ELF::NT_ARM_SVE;
   }
 }
 
@@ -1451,7 +1414,7 @@ Status NativeRegisterContextLinux_arm64::ReadPAuthMask() {
   ioVec.iov_base = GetPACMask();
   ioVec.iov_len = GetPACMaskSize();
 
-  error = ReadRegisterSet(&ioVec, GetPACMaskSize(), NT_ARM_PAC_MASK);
+  error = ReadRegisterSet(&ioVec, GetPACMaskSize(), llvm::ELF::NT_ARM_PAC_MASK);
 
   if (error.Success())
     m_pac_mask_is_valid = true;
@@ -1541,7 +1504,8 @@ Status NativeRegisterContextLinux_arm64::ReadMTEControl() {
   ioVec.iov_base = GetMTEControl();
   ioVec.iov_len = GetMTEControlSize();
 
-  error = ReadRegisterSet(&ioVec, GetMTEControlSize(), NT_ARM_TAGGED_ADDR_CTRL);
+  error = ReadRegisterSet(&ioVec, GetMTEControlSize(),
+                          llvm::ELF::NT_ARM_TAGGED_ADDR_CTRL);
 
   if (error.Success())
     m_mte_ctrl_is_valid = true;
@@ -1562,7 +1526,8 @@ Status NativeRegisterContextLinux_arm64::WriteMTEControl() {
 
   m_mte_ctrl_is_valid = false;
 
-  return WriteRegisterSet(&ioVec, GetMTEControlSize(), NT_ARM_TAGGED_ADDR_CTRL);
+  return WriteRegisterSet(&ioVec, GetMTEControlSize(),
+                          llvm::ELF::NT_ARM_TAGGED_ADDR_CTRL);
 }
 
 Status NativeRegisterContextLinux_arm64::ReadTLS() {
@@ -1575,7 +1540,7 @@ Status NativeRegisterContextLinux_arm64::ReadTLS() {
   ioVec.iov_base = GetTLSBuffer();
   ioVec.iov_len = GetTLSBufferSize();
 
-  error = ReadRegisterSet(&ioVec, GetTLSBufferSize(), NT_ARM_TLS);
+  error = ReadRegisterSet(&ioVec, GetTLSBufferSize(), llvm::ELF::NT_ARM_TLS);
 
   if (error.Success())
     m_tls_is_valid = true;
@@ -1596,7 +1561,7 @@ Status NativeRegisterContextLinux_arm64::WriteTLS() {
 
   m_tls_is_valid = false;
 
-  return WriteRegisterSet(&ioVec, GetTLSBufferSize(), NT_ARM_TLS);
+  return WriteRegisterSet(&ioVec, GetTLSBufferSize(), llvm::ELF::NT_ARM_TLS);
 }
 
 Status NativeRegisterContextLinux_arm64::ReadGCS() {
@@ -1609,7 +1574,7 @@ Status NativeRegisterContextLinux_arm64::ReadGCS() {
   ioVec.iov_base = GetGCSBuffer();
   ioVec.iov_len = GetGCSBufferSize();
 
-  error = ReadRegisterSet(&ioVec, GetGCSBufferSize(), NT_ARM_GCS);
+  error = ReadRegisterSet(&ioVec, GetGCSBufferSize(), llvm::ELF::NT_ARM_GCS);
 
   if (error.Success())
     m_gcs_is_valid = true;
@@ -1630,7 +1595,7 @@ Status NativeRegisterContextLinux_arm64::WriteGCS() {
 
   m_gcs_is_valid = false;
 
-  return WriteRegisterSet(&ioVec, GetGCSBufferSize(), NT_ARM_GCS);
+  return WriteRegisterSet(&ioVec, GetGCSBufferSize(), llvm::ELF::NT_ARM_GCS);
 }
 
 Status NativeRegisterContextLinux_arm64::ReadZAHeader() {
@@ -1643,7 +1608,7 @@ Status NativeRegisterContextLinux_arm64::ReadZAHeader() {
   ioVec.iov_base = GetZAHeader();
   ioVec.iov_len = GetZAHeaderSize();
 
-  error = ReadRegisterSet(&ioVec, GetZAHeaderSize(), NT_ARM_ZA);
+  error = ReadRegisterSet(&ioVec, GetZAHeaderSize(), llvm::ELF::NT_ARM_ZA);
 
   if (error.Success())
     m_za_header_is_valid = true;
@@ -1661,7 +1626,7 @@ Status NativeRegisterContextLinux_arm64::ReadZA() {
   ioVec.iov_base = GetZABuffer();
   ioVec.iov_len = GetZABufferSize();
 
-  error = ReadRegisterSet(&ioVec, GetZABufferSize(), NT_ARM_ZA);
+  error = ReadRegisterSet(&ioVec, GetZABufferSize(), llvm::ELF::NT_ARM_ZA);
 
   if (error.Success())
     m_za_buffer_is_valid = true;
@@ -1688,7 +1653,7 @@ Status NativeRegisterContextLinux_arm64::WriteZA() {
   // Writing to ZA may enable ZA, which means ZT0 may change too.
   m_zt_buffer_is_valid = false;
 
-  return WriteRegisterSet(&ioVec, GetZABufferSize(), NT_ARM_ZA);
+  return WriteRegisterSet(&ioVec, GetZABufferSize(), llvm::ELF::NT_ARM_ZA);
 }
 
 Status NativeRegisterContextLinux_arm64::ReadZT() {
@@ -1701,7 +1666,7 @@ Status NativeRegisterContextLinux_arm64::ReadZT() {
   ioVec.iov_base = GetZTBuffer();
   ioVec.iov_len = GetZTBufferSize();
 
-  error = ReadRegisterSet(&ioVec, GetZTBufferSize(), NT_ARM_ZT);
+  error = ReadRegisterSet(&ioVec, GetZTBufferSize(), llvm::ELF::NT_ARM_ZT);
   m_zt_buffer_is_valid = error.Success();
 
   return error;
@@ -1724,7 +1689,7 @@ Status NativeRegisterContextLinux_arm64::WriteZT() {
   m_za_buffer_is_valid = false;
   m_za_header_is_valid = false;
 
-  return WriteRegisterSet(&ioVec, GetZTBufferSize(), NT_ARM_ZT);
+  return WriteRegisterSet(&ioVec, GetZTBufferSize(), llvm::ELF::NT_ARM_ZT);
 }
 
 Status NativeRegisterContextLinux_arm64::ReadFPMR() {
@@ -1737,7 +1702,7 @@ Status NativeRegisterContextLinux_arm64::ReadFPMR() {
   ioVec.iov_base = GetFPMRBuffer();
   ioVec.iov_len = GetFPMRBufferSize();
 
-  error = ReadRegisterSet(&ioVec, GetFPMRBufferSize(), NT_ARM_FPMR);
+  error = ReadRegisterSet(&ioVec, GetFPMRBufferSize(), llvm::ELF::NT_ARM_FPMR);
 
   if (error.Success())
     m_fpmr_is_valid = true;
@@ -1758,7 +1723,7 @@ Status NativeRegisterContextLinux_arm64::WriteFPMR() {
 
   m_fpmr_is_valid = false;
 
-  return WriteRegisterSet(&ioVec, GetFPMRBufferSize(), NT_ARM_FPMR);
+  return WriteRegisterSet(&ioVec, GetFPMRBufferSize(), llvm::ELF::NT_ARM_FPMR);
 }
 
 Status NativeRegisterContextLinux_arm64::ReadPOE() {
@@ -1771,7 +1736,7 @@ Status NativeRegisterContextLinux_arm64::ReadPOE() {
   ioVec.iov_base = GetPOEBuffer();
   ioVec.iov_len = GetPOEBufferSize();
 
-  error = ReadRegisterSet(&ioVec, GetPOEBufferSize(), NT_ARM_POE);
+  error = ReadRegisterSet(&ioVec, GetPOEBufferSize(), llvm::ELF::NT_ARM_POE);
 
   if (error.Success())
     m_poe_is_valid = true;
@@ -1792,7 +1757,7 @@ Status NativeRegisterContextLinux_arm64::WritePOE() {
 
   m_poe_is_valid = false;
 
-  return WriteRegisterSet(&ioVec, GetPOEBufferSize(), NT_ARM_POE);
+  return WriteRegisterSet(&ioVec, GetPOEBufferSize(), llvm::ELF::NT_ARM_POE);
 }
 
 void NativeRegisterContextLinux_arm64::ConfigureRegisterContext() {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index b2a363231237f..f4d3f87331a88 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -2478,9 +2478,17 @@ Function *DWARFASTParserClang::ParseFunctionFromDWARF(
                                decl_line, decl_column, call_file, call_line,
                                call_column, &frame_base)) {
     Mangled func_name;
-    if (mangled)
+    if (mangled && name &&
+        Mangled::GetManglingScheme(mangled) == Mangled::eManglingSchemeNone) {
+      // The linkage name is present but is not actually a mangled name (e.g.
+      // wasi-libc renames `main` to its `__main_argc_argv` argv-passing
+      // wrapper). Display the source name (DW_AT_name) and keep the linkage
+      // name as the symbol so lookups by either name still resolve.
+      func_name.SetDemangledName(ConstString(name));
+      func_name.SetMangledName(ConstString(mangled));
+    } else if (mangled) {
       func_name.SetValue(ConstString(mangled));
-    else if ((die.GetParent().Tag() == DW_TAG_compile_unit ||
+    } else if ((die.GetParent().Tag() == DW_TAG_compile_unit ||
               die.GetParent().Tag() == DW_TAG_partial_unit) &&
              Language::LanguageIsCPlusPlus(
                  SymbolFileDWARF::GetLanguage(*die.GetCU())) &&
diff --git a/lldb/source/Utility/VirtualDataExtractor.cpp b/lldb/source/Utility/VirtualDataExtractor.cpp
index fe7ed963d0bdd..167560bb08344 100644
--- a/lldb/source/Utility/VirtualDataExtractor.cpp
+++ b/lldb/source/Utility/VirtualDataExtractor.cpp
@@ -246,24 +246,30 @@ uint64_t VirtualDataExtractor::GetU64_unchecked(offset_t *offset_ptr) const {
   return result;
 }
 
+/// A subset request that can't be mapped still returns a valid, non-null
+/// extractor, because most callers dereference the result directly.
+static DataExtractorSP MakeEmptySubsetExtractor(ByteOrder byte_order,
+                                                uint32_t addr_size) {
+  auto empty_sp = std::make_shared<DataExtractor>();
+  empty_sp->SetByteOrder(byte_order);
+  empty_sp->SetAddressByteSize(addr_size);
+  return empty_sp;
+}
+
 DataExtractorSP
 VirtualDataExtractor::GetSubsetExtractorSP(offset_t virtual_offset,
                                            offset_t virtual_length) {
   const LookupTable::Entry *entry = FindEntry(virtual_offset);
-  assert(
-      entry &&
-      "VirtualDataExtractor subset extractor requires valid virtual address");
   if (!entry)
-    return {};
+    return MakeEmptySubsetExtractor(GetByteOrder(), GetAddressByteSize());
 
   // Entry::data is the offset into the DataBuffer's actual start/end range
   // Entry::base is the virtual address at the start of this region of data
   offset_t offset_into_entry_range = virtual_offset - entry->base;
-  assert(
-      offset_into_entry_range + virtual_length <= entry->size &&
-      "VirtualDataExtractor subset may not span multiple LookupTable entries");
+  // A subset may not span more than one entry; a range that overflows this
+  // entry is unmappable.
   if (offset_into_entry_range + virtual_length > entry->size)
-    return {};
+    return MakeEmptySubsetExtractor(GetByteOrder(), GetAddressByteSize());
 
   // We could support a Subset VirtualDataExtractor which covered
   // multiple LookupTable virtual entries, but we'd need to mutate
@@ -283,11 +289,8 @@ VirtualDataExtractor::GetSubsetExtractorSP(offset_t virtual_offset,
 DataExtractorSP
 VirtualDataExtractor::GetSubsetExtractorSP(offset_t virtual_offset) {
   const LookupTable::Entry *entry = FindEntry(virtual_offset);
-  assert(
-      entry &&
-      "VirtualDataExtractor subset extractor requires valid virtual address");
   if (!entry)
-    return {};
+    return MakeEmptySubsetExtractor(GetByteOrder(), GetAddressByteSize());
 
   // Entry::data is the offset into the DataBuffer's actual start/end range
   // Entry::base is the virtual address at the start of this region of data
diff --git a/lldb/test/Shell/SymbolFile/DWARF/wasm-function-display-name.yaml b/lldb/test/Shell/SymbolFile/DWARF/wasm-function-display-name.yaml
new file mode 100644
index 0000000000000..9a0df8642403f
--- /dev/null
+++ b/lldb/test/Shell/SymbolFile/DWARF/wasm-function-display-name.yaml
@@ -0,0 +1,81 @@
+# A function whose DWARF DW_AT_linkage_name is present but is NOT a mangled
+# name -- it differs from DW_AT_name. This happens on WebAssembly, where
+# wasi-libc renames `main` to its `__main_argc_argv` argv-passing wrapper:
+# the DWARF keeps DW_AT_name "main" but records DW_AT_linkage_name
+# "__main_argc_argv". Here the same shape is produced with an asm() rename.
+#
+# LLDB must display the source name (DW_AT_name, "impl") rather than the raw,
+# un-demanglable linkage name, while still keeping the linkage name as the
+# symbol so a breakpoint set by either name resolves.
+
+# RUN: yaml2obj %s -o %t
+# RUN: %lldb %t \
+# RUN:   -o "image lookup -v -n impl" \
+# RUN:   -o "breakpoint set -n impl" \
+# RUN:   -o exit 2>&1 | FileCheck %s
+
+# The function is displayed by its source name, with the linkage name kept as
+# the symbol; a breakpoint set by the source name resolves and displays it too.
+# CHECK: Function: {{.*}}name = "impl", mangled = "renamed_export"
+# CHECK: Breakpoint 1: where = {{.*}}`impl
+
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:
+          - I32
+        ReturnTypes:
+          - I32
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0 ]
+  - Type:            MEMORY
+    Memories:
+      - Minimum:         0x1
+  - Type:            GLOBAL
+    Globals:
+      - Index:           0
+        Type:            I32
+        Mutable:         true
+        InitExpr:
+          Opcode:          I32_CONST
+          Value:           65536
+  - Type:            EXPORT
+    Exports:
+      - Name:            memory
+        Kind:            MEMORY
+        Index:           0
+      - Name:            renamed_export
+        Kind:            FUNCTION
+        Index:           0
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:
+          - Type:            I32
+            Count:           1
+        Body:            23808080800041106B21012001200036020C200128020C41016A0F0B
+  - Type:            CUSTOM
+    Name:            .debug_abbrev
+    Payload:         011101250E1305030E10171B0E110112060000022E011101120640186E0E030E3A0B3B0B271949133F1900000305000218030E3A0B3B0B49130000042400030E3E0B0B0B000000
+  - Type:            CUSTOM
+    Name:            .debug_info
+    Payload:         5500000004000000000004012D0000001D001F0000000000000015000000020000001F00000002020000001F00000004ED00019F020000001A0000000104510000000302910C00000000010451000000000411000000050400
+  - Type:            CUSTOM
+    Name:            .debug_str
+    Payload:         780072656E616D65645F6578706F727400696E74002F746D7000696D706C002F746D702F72656E616D652E6300636C616E672076657273696F6E2032322E312E302D776173692D73646B202868747470733A2F2F6769746875622E636F6D2F6C6C766D2F6C6C766D2D70726F6A65637420343433346461626236393931363835366238323466363861363462303239633637313735653533322900
+  - Type:            CUSTOM
+    Name:            .debug_line
+    Payload:         3F000000040020000000010101FB0E0D0001010101000000010000010072656E616D652E6300000000000005020200000015051A0A084A051C065805133C0202000101
+  - Type:            CUSTOM
+    Name:            name
+    FunctionNames:
+      - Index:           0
+        Name:            renamed_export
+    GlobalNames:
+      - Index:           0
+        Name:            __stack_pointer
+...
diff --git a/lldb/unittests/Utility/VirtualDataExtractorTest.cpp b/lldb/unittests/Utility/VirtualDataExtractorTest.cpp
index 5a1ec9d6332de..89c1f27fdbc87 100644
--- a/lldb/unittests/Utility/VirtualDataExtractorTest.cpp
+++ b/lldb/unittests/Utility/VirtualDataExtractorTest.cpp
@@ -661,3 +661,40 @@ TEST(VirtualDataExtractorTest, SubsetExtractorGetU32) {
   // Entry(0x0, 4*sizeof(uint32_t), 12*sizeof(uint32_t))
   EXPECT_EQ(extractor->GetU32(&virtual_offset), 0xccccccccU);
 }
+
+TEST(VirtualDataExtractorTest, SubsetExtractorUnmappedRangeReturnsEmpty) {
+  // Two mapped regions with a gap in the virtual address space between them.
+  uint8_t buffer[] = {0x01, 0x02, 0x03, 0x04, 0x11, 0x12, 0x13, 0x14};
+  DataBufferSP buffer_sp =
+      std::make_shared<DataBufferUnowned>(buffer, sizeof(buffer));
+  lldb::DataExtractorSP extractor = std::make_shared<VirtualDataExtractor>(
+      buffer_sp, eByteOrderLittle, 8,
+      Table{Entry(0x1000, 4, 0), Entry(0x2000, 4, 4)});
+
+  // An offset in the gap between entries must not produce a null shared_ptr --
+  // callers dereference the result -- so it returns a valid empty extractor.
+  lldb::DataExtractorSP gap_subset = extractor->GetSubsetExtractorSP(0x1800, 4);
+  ASSERT_NE(gap_subset, nullptr);
+  EXPECT_EQ(gap_subset->GetByteSize(), 0U);
+
+  // A range starting inside an entry but running past its end (into the
+  // unmapped gap) is likewise unsatisfiable and returns an empty extractor.
+  lldb::DataExtractorSP overrun_subset =
+      extractor->GetSubsetExtractorSP(0x1002, 8);
+  ASSERT_NE(overrun_subset, nullptr);
+  EXPECT_EQ(overrun_subset->GetByteSize(), 0U);
+
+  // The single-argument overload behaves the same for an unmapped offset.
+  lldb::DataExtractorSP gap_subset_single =
+      extractor->GetSubsetExtractorSP(0x1800);
+  ASSERT_NE(gap_subset_single, nullptr);
+  EXPECT_EQ(gap_subset_single->GetByteSize(), 0U);
+
+  // A fully-contained range still produces a real subset with the right bytes.
+  lldb::DataExtractorSP valid_subset =
+      extractor->GetSubsetExtractorSP(0x2000, 4);
+  ASSERT_NE(valid_subset, nullptr);
+  EXPECT_EQ(valid_subset->GetByteSize(), 4U);
+  offset_t physical_offset = 0;
+  EXPECT_EQ(valid_subset->GetU32(&physical_offset), 0x14131211U);
+}
diff --git a/llvm/docs/CommandGuide/index.md b/llvm/docs/CommandGuide/index.md
index 6b126837343e2..ac90d9468b848 100644
--- a/llvm/docs/CommandGuide/index.md
+++ b/llvm/docs/CommandGuide/index.md
@@ -1,104 +1,149 @@
 # LLVM Command Guide
 
-The following documents are command descriptions for all of the LLVM tools.
-These pages describe how to use the LLVM commands and what their options are.
-Note that these pages do not describe all of the options available for all
-tools. To get a complete listing, pass the `--help` (general options) or
-`--help-hidden` (general and debugging options) arguments to the tool you are
-interested in.
-
-## Basic Commands
-
 ```{toctree}
-:maxdepth: 1
+:hidden:
 
+FileCheck
+clang-tblgen
 dsymutil
+lit
 llc
+lldb-tblgen
 lli
 llubi
+llvm-addr2line
+llvm-ar
 llvm-as
+llvm-bcanalyzer
 llvm-cgdata
 llvm-config
 llvm-cov
+llvm-cxxfilt
 llvm-cxxmap
 llvm-debuginfo-analyzer
 llvm-diff
 llvm-dis
 llvm-dwarfdump
 llvm-dwarfutil
+llvm-exegesis
+llvm-extract
 llvm-extract-bundle-entry
+llvm-ifs
+llvm-install-name-tool
 llvm-ir2vec
 llvm-lib
 llvm-libtool-darwin
 llvm-link
 llvm-lipo
+llvm-locstats
 llvm-mc
 llvm-mca
+llvm-nm
+llvm-objcopy
+llvm-objdump
+llvm-offload-binary
 llvm-opt-report
 llvm-otool
+llvm-pdbutil
 llvm-profdata
+llvm-profgen
+llvm-ranlib
+llvm-readelf
 llvm-readobj
 llvm-reduce
+llvm-remarkutil
+llvm-size
 llvm-stress
+llvm-strings
+llvm-strip
 llvm-symbolizer
+llvm-tblgen
+llvm-test-mustache-spec
+llvm-tli-checker
+mlir-tblgen
 opt
+tblgen
 ```
 
-## GNU binutils replacements
+The following documents are command descriptions for all of the LLVM tools.
+These pages describe how to use the LLVM commands and what their options are.
+Note that these pages do not describe all of the options available for all
+tools. To get a complete listing, pass the `--help` (general options) or
+`--help-hidden` (general and debugging options) arguments to the tool you are
+interested in.
 
-```{toctree}
-:maxdepth: 1
+## Basic Commands
 
-llvm-addr2line
-llvm-ar
-llvm-cxxfilt
-llvm-install-name-tool
-llvm-nm
-llvm-objcopy
-llvm-objdump
-llvm-ranlib
-llvm-readelf
-llvm-size
-llvm-strings
-llvm-strip
-```
+* {doc}`dsymutil`
+* {doc}`llc`
+* {doc}`lli`
+* {doc}`llubi`
+* {doc}`llvm-as`
+* {doc}`llvm-cgdata`
+* {doc}`llvm-config`
+* {doc}`llvm-cov`
+* {doc}`llvm-cxxmap`
+* {doc}`llvm-debuginfo-analyzer`
+* {doc}`llvm-diff`
+* {doc}`llvm-dis`
+* {doc}`llvm-dwarfdump`
+* {doc}`llvm-dwarfutil`
+* {doc}`llvm-extract-bundle-entry`
+* {doc}`llvm-ir2vec`
+* {doc}`llvm-lib`
+* {doc}`llvm-libtool-darwin`
+* {doc}`llvm-link`
+* {doc}`llvm-lipo`
+* {doc}`llvm-mc`
+* {doc}`llvm-mca`
+* {doc}`llvm-opt-report`
+* {doc}`llvm-otool`
+* {doc}`llvm-profdata`
+* {doc}`llvm-readobj`
+* {doc}`llvm-reduce`
+* {doc}`llvm-stress`
+* {doc}`llvm-symbolizer`
+* {doc}`opt`
 
-## Debugging Tools
+## GNU binutils replacements
 
-```{toctree}
-:maxdepth: 1
+* {doc}`llvm-addr2line`
+* {doc}`llvm-ar`
+* {doc}`llvm-cxxfilt`
+* {doc}`llvm-install-name-tool`
+* {doc}`llvm-nm`
+* {doc}`llvm-objcopy`
+* {doc}`llvm-objdump`
+* {doc}`llvm-ranlib`
+* {doc}`llvm-readelf`
+* {doc}`llvm-size`
+* {doc}`llvm-strings`
+* {doc}`llvm-strip`
 
-llvm-extract
-llvm-bcanalyzer
-llvm-reduce
-```
+## Debugging Tools
+
+* {doc}`llvm-extract`
+* {doc}`llvm-bcanalyzer`
+* {doc}`llvm-reduce`
 
 ## Developer Tools
 
-```{toctree}
-:maxdepth: 1
-
-FileCheck
-tblgen
-clang-tblgen
-lldb-tblgen
-llvm-tblgen
-mlir-tblgen
-lit
-llvm-exegesis
-llvm-ifs
-llvm-locstats
-llvm-test-mustache-spec
-llvm-pdbutil
-llvm-profgen
-llvm-tli-checker
-llvm-offload-binary
-```
+* {doc}`FileCheck`
+* {doc}`tblgen`
+* {doc}`clang-tblgen`
+* {doc}`lldb-tblgen`
+* {doc}`llvm-tblgen`
+* {doc}`mlir-tblgen`
+* {doc}`lit`
+* {doc}`llvm-exegesis`
+* {doc}`llvm-ifs`
+* {doc}`llvm-locstats`
+* {doc}`llvm-test-mustache-spec`
+* {doc}`llvm-pdbutil`
+* {doc}`llvm-profgen`
+* {doc}`llvm-tli-checker`
+* {doc}`llvm-offload-binary`
 
 ## Remarks Tools
 
-```{toctree}
-:maxdepth: 1
-
-llvm-remarkutil
-```
+* {doc}`llvm-remarkutil`
diff --git a/llvm/docs/GettingInvolved.md b/llvm/docs/GettingInvolved.md
index a172481f81df9..c7d4584af4770 100644
--- a/llvm/docs/GettingInvolved.md
+++ b/llvm/docs/GettingInvolved.md
@@ -17,7 +17,6 @@ SphinxQuickstartTemplate
 HowToSubmitABug
 BugLifeCycle
 CodingStandards
-GitHub
 GitBisecting
 GitRepositoryPolicy
 ```
diff --git a/llvm/docs/GlobalISel/Pipeline.rst b/llvm/docs/GlobalISel/Pipeline.rst
index b9085e82f1428..8565c8c95178b 100644
--- a/llvm/docs/GlobalISel/Pipeline.rst
+++ b/llvm/docs/GlobalISel/Pipeline.rst
@@ -3,14 +3,6 @@
 Core Pipeline
 =============
 
-.. toctree::
-  :hidden:
-
-  IRTranslator
-  Legalizer
-  RegBankSelect
-  InstructionSelect
-
 The core pipeline of GlobalISel is:
 
 .. image:: pipeline-overview.png
diff --git a/llvm/docs/GlobalISel/Porting.rst b/llvm/docs/GlobalISel/Porting.rst
index 0bc5e7446b83f..89aeeb5f53fcb 100644
--- a/llvm/docs/GlobalISel/Porting.rst
+++ b/llvm/docs/GlobalISel/Porting.rst
@@ -21,7 +21,7 @@ Additionally:
   including additional passes not included in the :ref:`pipeline`.
 
 Tutorials
-=========
+---------
 
 We'd recommend watching `this tutorial
 <https://www.llvm.org/devmtg/2017-10/#tutorial2>`_ from the 2017 LLVM DevMeeting
diff --git a/llvm/docs/GlobalISel/index.rst b/llvm/docs/GlobalISel/index.rst
index 2b639706afdc2..4951e09c257fd 100644
--- a/llvm/docs/GlobalISel/index.rst
+++ b/llvm/docs/GlobalISel/index.rst
@@ -10,8 +10,23 @@ Global Instruction Selection
    :local:
    :depth: 1
 
+.. toctree::
+   :hidden:
+
+   GMIR
+   GenericOpcode
+   MIRPatterns
+   Pipeline
+   Porting
+   Resources
+   IRTranslator
+   Legalizer
+   RegBankSelect
+   InstructionSelect
+   KnownBits
+
 Introduction
-============
+------------
 
 GlobalISel is a framework that provides a set of reusable passes and utilities
 for instruction selection --- translation from LLVM IR to target-specific
@@ -41,36 +56,30 @@ solve three major problems:
   configure that pipeline to better suit their needs.
 
 Design and Implementation Reference
-===================================
+-----------------------------------
 
 More information on the design and implementation of GlobalISel can be found in
 the following sections.
 
-.. toctree::
-  :maxdepth: 1
-
-  GMIR
-  GenericOpcode
-  MIRPatterns
-  Pipeline
-  Porting
-  Resources
+* :doc:`GMIR`
+* :doc:`GenericOpcode`
+* :doc:`MIRPatterns`
+* :doc:`Pipeline`
+* :doc:`Porting`
+* :doc:`Resources`
 
 More information on specific passes can be found in the following sections:
 
-.. toctree::
-  :maxdepth: 1
-
-  IRTranslator
-  Legalizer
-  RegBankSelect
-  InstructionSelect
-  KnownBits
+* :doc:`IRTranslator`
+* :doc:`Legalizer`
+* :doc:`RegBankSelect`
+* :doc:`InstructionSelect`
+* :doc:`KnownBits`
 
 .. _progress:
 
 Progress and Future Work
-========================
+------------------------
 
 The initial goal is to replace FastISel on AArch64.  The next step will be to
 replace SelectionDAG as the optimized ISel.
@@ -84,7 +93,7 @@ that is destroyed after :ref:`instructionselect`.
 .. _progress-fastisel:
 
 FastISel Replacement
---------------------
+^^^^^^^^^^^^^^^^^^^^
 
 For the initial FastISel replacement, we intend to fallback to SelectionDAG on
 selection failures.
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index f2839eab14d0c..eb0e7fd685916 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -2390,6 +2390,14 @@ For example:
     This attribute indicates that the inliner should never inline this
     function in any situation. This attribute may not be used together
     with the ``alwaysinline`` attribute.
+``noipa``
+    Disables any interprocedural analysis that inspects the definition of this
+    function. This attribute is equivalent to moving this function definition to
+    a separate, optimizer-opaque, module. Any attributes on the function are
+    still respected (as they would be if they remained on a function declaration
+    in this module). This attribute does *not* control inlining or outlining.
+    Add the ``noinline`` and ``nooutline`` attributes as well in cases where
+    inlining and outlining should additionally be disabled.
 ``nomerge``
     This attribute indicates that calls to this function should never be merged
     during optimization. For example, it will prevent tail merging otherwise
diff --git a/llvm/docs/Reference.md b/llvm/docs/Reference.md
index cddcc9056128d..1bbdc8f4582dd 100644
--- a/llvm/docs/Reference.md
+++ b/llvm/docs/Reference.md
@@ -11,7 +11,6 @@ LLVM and API reference documentation.
 
 HowToUseAttributes
 CommandGuide/index
-CommandGuide/llvm-reduce
 OptBisect
 SymbolizerMarkupFormat
 PDB/index
@@ -27,7 +26,6 @@ MIRLangRef
 GlobalISel/index
 ConvergentOperations
 TestingGuide
-TestSuiteGuide
 GwpAsan
 XRay
 XRayExample
@@ -148,7 +146,6 @@ XRayFDRFormat
 {doc}`TestSuiteGuide`
 :   Describes how to compile and run the test-suite benchmarks.
 
-
 {doc}`GwpAsan`
 :   A sampled heap memory error detection toolkit designed for production use.
 
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index a365e30792b4c..24b8604ca11d8 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -97,6 +97,11 @@ Makes programs 10x faster by doing Special New Thing.
 * The ``modular-format`` attribute now supports the ``fixed`` aspect for C
   ISO 18037 fixed-point ``printf`` specifiers.
 
+* Added `noipa` attribute which disables interprocedural analyses that inspect
+  the definition of the function. This attribute does *not* control inlining or
+  outlining. Add the `noinline` and `nooutline` attributes as well in cases
+  where inlining and outlining should additionally be disabled.
+
 ### Changes to LLVM infrastructure
 
 * Removed ``Constant::isZeroValue``. It was functionally identical to
diff --git a/llvm/docs/UserGuides.md b/llvm/docs/UserGuides.md
index 72ebd3b9cedce..e91cdd8975115 100644
--- a/llvm/docs/UserGuides.md
+++ b/llvm/docs/UserGuides.md
@@ -48,7 +48,6 @@ InstCombineContributorGuide
 WritingAnLLVMBackend
 CodeGenerator
 TableGen/index
-GlobalISel/MIRPatterns
 MCJITDesignAndImplementation
 ORCv2
 JITLink
@@ -64,8 +63,6 @@ CompileCudaWithLLVM
 NVPTXUsage
 AMDGPUUsage
 AMDGPUAsyncOperations
-AMDGPUDwarfExtensionsForHeterogeneousDebugging
-AMDGPUDwarfExtensionAllowLocationDescriptionOnTheDwarfExpressionStack/AMDGPUDwarfExtensionAllowLocationDescriptionOnTheDwarfExpressionStack
 AMDGPUExecutionSynchronization
 AMDGPUMemoryModel
 SPIRVUsage
diff --git a/llvm/docs/conf.py b/llvm/docs/conf.py
index 2727f2248e803..8a5d0186250fd 100644
--- a/llvm/docs/conf.py
+++ b/llvm/docs/conf.py
@@ -22,7 +22,7 @@
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions += ["sphinx.ext.intersphinx", "sphinx.ext.todo"]
+extensions += ["sphinx.ext.intersphinx", "sphinx.ext.todo", "llvm_sphinx.ext.checks"]
 
 myst_enable_extensions += ["deflist"]
 myst_url_schemes = {
diff --git a/llvm/docs/tutorial/MyFirstLanguageFrontend/index.rst b/llvm/docs/tutorial/MyFirstLanguageFrontend/index.rst
index 669df43111cc7..230313ddd21db 100644
--- a/llvm/docs/tutorial/MyFirstLanguageFrontend/index.rst
+++ b/llvm/docs/tutorial/MyFirstLanguageFrontend/index.rst
@@ -2,20 +2,6 @@
 My First Language Frontend with LLVM Tutorial
 =============================================
 
-.. toctree::
-   :hidden:
-
-   LangImpl01
-   LangImpl02
-   LangImpl03
-   LangImpl04
-   LangImpl05
-   LangImpl06
-   LangImpl07
-   LangImpl08
-   LangImpl09
-   LangImpl10
-
 **Requirements:** This tutorial assumes you know C++, but no previous
 compiler experience is necessary.
 
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index 9ae3f15350780..b94c64957c4e3 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -823,6 +823,7 @@ enum AttributeKindCodes {
   ATTR_KIND_DENORMAL_FPENV = 106,
   ATTR_KIND_NOOUTLINE = 107,
   ATTR_KIND_FLATTEN = 108,
+  ATTR_KIND_NOIPA = 109,
 
   // TODO: Get rid of this.
   // There really shouldn't be incompatible bitcode changes specific to AMD
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index 0f266fed9dcee..05cad63f93aad 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -218,6 +218,9 @@ def NoInline : EnumAttr<"noinline", IntersectPreserve, [FnAttr]>;
 /// nooutline
 def NoOutline : EnumAttr<"nooutline", IntersectPreserve, [FnAttr]>;
 
+/// Do not do interprocedural analysis or optimization including this function
+def NoIPA : EnumAttr<"noipa", IntersectPreserve, [FnAttr]>;
+
 /// Function is called early and/or often, so lazy binding isn't worthwhile.
 def NonLazyBind : EnumAttr<"nonlazybind", IntersectPreserve, [FnAttr]>;
 
diff --git a/llvm/include/llvm/IR/GlobalValue.h b/llvm/include/llvm/IR/GlobalValue.h
index 83e695cdd27d9..7a23c1cb00a14 100644
--- a/llvm/include/llvm/IR/GlobalValue.h
+++ b/llvm/include/llvm/IR/GlobalValue.h
@@ -165,6 +165,10 @@ class GlobalValue : public Constant {
   /// attribute.
   LLVM_ABI bool isNobuiltinFnDef() const;
 
+  /// Returns true if the global is a function definition with the noipa
+  /// attribute.
+  LLVM_ABI bool isNoipaFnDef() const;
+
 protected:
   /// The intrinsic ID for this subclass (which must be a Function).
   ///
@@ -506,8 +510,12 @@ class GlobalValue : public Constant {
   /// Return true if this global's definition can be substituted with an
   /// *arbitrary* definition at link time or load time. We cannot do any IPO or
   /// inlining across interposable call edges, since the callee can be
-  /// replaced with something arbitrary.
-  LLVM_ABI bool isInterposable() const;
+  /// replaced with something arbitrary. For most IPO passes, the `noipa`
+  /// attribute on a function definition is also treated as if it were
+  /// interposable (and thus blocking interprocedural analysis). Passes
+  /// which already have their own distinct control attributes (e.g. inlining)
+  /// may set `CheckNoIPA = false` when calling this.
+  LLVM_ABI bool isInterposable(bool CheckNoIPA = true) const;
   LLVM_ABI bool canBenefitFromLocalAlias() const;
 
   bool hasExternalLinkage() const { return isExternalLinkage(getLinkage()); }
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 8155ae3258a12..cc7611fb3aba7 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -584,7 +584,7 @@ def llvm_anyvector_ty  : LLVMAnyType<vAny,
 
 // Note: CodeGenIntrinsics.cpp seems to check this class to check pointers.
 class LLVMAnyPointerType : LLVMAnyType<pAny,
-                                       AnyKindVectorConstraint.None,
+                                       AnyKindVectorConstraint.Scalar,
                                        AnyKindElementConstraint.Pointer>;
 
 def llvm_anyptr_ty     : LLVMAnyPointerType;      // ptr addrspace(N)
diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h
index 4ff324deeb026..1b19175156261 100644
--- a/llvm/include/llvm/IR/PatternMatch.h
+++ b/llvm/include/llvm/IR/PatternMatch.h
@@ -2447,37 +2447,25 @@ m_Br(const Cond_t &C, const TrueBlock_t &T, const FalseBlock_t &F) {
 }
 
 //===----------------------------------------------------------------------===//
-// Matchers for max/min idioms, eg: "select (sgt x, y), x, y" -> smax(x,y).
+// Matchers for fmax/fmin idioms, eg: "select (sgt x, y), x, y" -> smax(x,y).
 //
 
-template <typename CmpInst_t, typename LHS_t, typename RHS_t, typename Pred_t,
-          bool Commutable = false>
-struct MaxMin_match {
+template <typename LHS_t, typename RHS_t, typename Pred_t>
+struct FMaxMin_match {
   using PredType = Pred_t;
   LHS_t L;
   RHS_t R;
 
   // The evaluation order is always stable, regardless of Commutability.
   // The LHS is always matched first.
-  MaxMin_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {}
+  FMaxMin_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {}
 
   template <typename OpTy> bool match(OpTy *V) const {
-    if (auto *II = dyn_cast<IntrinsicInst>(V)) {
-      Intrinsic::ID IID = II->getIntrinsicID();
-      if ((IID == Intrinsic::smax && Pred_t::match(ICmpInst::ICMP_SGT)) ||
-          (IID == Intrinsic::smin && Pred_t::match(ICmpInst::ICMP_SLT)) ||
-          (IID == Intrinsic::umax && Pred_t::match(ICmpInst::ICMP_UGT)) ||
-          (IID == Intrinsic::umin && Pred_t::match(ICmpInst::ICMP_ULT))) {
-        Value *LHS = II->getOperand(0), *RHS = II->getOperand(1);
-        return (L.match(LHS) && R.match(RHS)) ||
-               (Commutable && L.match(RHS) && R.match(LHS));
-      }
-    }
     // Look for "(x pred y) ? x : y" or "(x pred y) ? y : x".
     auto *SI = dyn_cast<SelectInst>(V);
     if (!SI)
       return false;
-    auto *Cmp = dyn_cast<CmpInst_t>(SI->getCondition());
+    auto *Cmp = dyn_cast<FCmpInst>(SI->getCondition());
     if (!Cmp)
       return false;
     // At this point we have a select conditioned on a comparison.  Check that
@@ -2489,42 +2477,13 @@ struct MaxMin_match {
     if ((TrueVal != LHS || FalseVal != RHS) &&
         (TrueVal != RHS || FalseVal != LHS))
       return false;
-    typename CmpInst_t::Predicate Pred =
+    FCmpInst::Predicate Pred =
         LHS == TrueVal ? Cmp->getPredicate() : Cmp->getInversePredicate();
     // Does "(x pred y) ? x : y" represent the desired max/min operation?
     if (!Pred_t::match(Pred))
       return false;
     // It does!  Bind the operands.
-    return (L.match(LHS) && R.match(RHS)) ||
-           (Commutable && L.match(RHS) && R.match(LHS));
-  }
-};
-
-/// Helper class for identifying signed max predicates.
-struct smax_pred_ty {
-  static bool match(ICmpInst::Predicate Pred) {
-    return Pred == CmpInst::ICMP_SGT || Pred == CmpInst::ICMP_SGE;
-  }
-};
-
-/// Helper class for identifying signed min predicates.
-struct smin_pred_ty {
-  static bool match(ICmpInst::Predicate Pred) {
-    return Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_SLE;
-  }
-};
-
-/// Helper class for identifying unsigned max predicates.
-struct umax_pred_ty {
-  static bool match(ICmpInst::Predicate Pred) {
-    return Pred == CmpInst::ICMP_UGT || Pred == CmpInst::ICMP_UGE;
-  }
-};
-
-/// Helper class for identifying unsigned min predicates.
-struct umin_pred_ty {
-  static bool match(ICmpInst::Predicate Pred) {
-    return Pred == CmpInst::ICMP_ULT || Pred == CmpInst::ICMP_ULE;
+    return L.match(LHS) && R.match(RHS);
   }
 };
 
@@ -2556,35 +2515,6 @@ struct ufmin_pred_ty {
   }
 };
 
-template <typename LHS, typename RHS>
-inline MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty> m_SMax(const LHS &L,
-                                                             const RHS &R) {
-  return MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty>(L, R);
-}
-
-template <typename LHS, typename RHS>
-inline MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty> m_SMin(const LHS &L,
-                                                             const RHS &R) {
-  return MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty>(L, R);
-}
-
-template <typename LHS, typename RHS>
-inline MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty> m_UMax(const LHS &L,
-                                                             const RHS &R) {
-  return MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty>(L, R);
-}
-
-template <typename LHS, typename RHS>
-inline MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty> m_UMin(const LHS &L,
-                                                             const RHS &R) {
-  return MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty>(L, R);
-}
-
-template <typename LHS, typename RHS>
-inline auto m_MaxOrMin(const LHS &L, const RHS &R) {
-  return m_CombineOr(m_SMax(L, R), m_SMin(L, R), m_UMax(L, R), m_UMin(L, R));
-}
-
 /// Match an 'ordered' floating point maximum function.
 /// Floating point has one special value 'NaN'. Therefore, there is no total
 /// order. However, if we can ignore the 'NaN' value (for example, because of a
@@ -2595,9 +2525,9 @@ inline auto m_MaxOrMin(const LHS &L, const RHS &R) {
 ///                         max(L, R)  iff L and R are not NaN
 ///  m_OrdFMax(L, R) =      R          iff L or R are NaN
 template <typename LHS, typename RHS>
-inline MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty> m_OrdFMax(const LHS &L,
-                                                                 const RHS &R) {
-  return MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty>(L, R);
+inline FMaxMin_match<LHS, RHS, ofmax_pred_ty> m_OrdFMax(const LHS &L,
+                                                        const RHS &R) {
+  return FMaxMin_match<LHS, RHS, ofmax_pred_ty>(L, R);
 }
 
 /// Match an 'ordered' floating point minimum function.
@@ -2610,9 +2540,9 @@ inline MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty> m_OrdFMax(const LHS &L,
 ///                         min(L, R)  iff L and R are not NaN
 ///  m_OrdFMin(L, R) =      R          iff L or R are NaN
 template <typename LHS, typename RHS>
-inline MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty> m_OrdFMin(const LHS &L,
-                                                                 const RHS &R) {
-  return MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty>(L, R);
+inline FMaxMin_match<LHS, RHS, ofmin_pred_ty> m_OrdFMin(const LHS &L,
+                                                        const RHS &R) {
+  return FMaxMin_match<LHS, RHS, ofmin_pred_ty>(L, R);
 }
 
 /// Match an 'unordered' floating point maximum function.
@@ -2625,9 +2555,9 @@ inline MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty> m_OrdFMin(const LHS &L,
 ///                         max(L, R)  iff L and R are not NaN
 ///  m_UnordFMax(L, R) =    L          iff L or R are NaN
 template <typename LHS, typename RHS>
-inline MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty>
-m_UnordFMax(const LHS &L, const RHS &R) {
-  return MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty>(L, R);
+inline FMaxMin_match<LHS, RHS, ufmax_pred_ty> m_UnordFMax(const LHS &L,
+                                                          const RHS &R) {
+  return FMaxMin_match<LHS, RHS, ufmax_pred_ty>(L, R);
 }
 
 /// Match an 'unordered' floating point minimum function.
@@ -2640,9 +2570,9 @@ m_UnordFMax(const LHS &L, const RHS &R) {
 ///                          min(L, R)  iff L and R are not NaN
 ///  m_UnordFMin(L, R) =     L          iff L or R are NaN
 template <typename LHS, typename RHS>
-inline MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty>
-m_UnordFMin(const LHS &L, const RHS &R) {
-  return MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty>(L, R);
+inline FMaxMin_match<LHS, RHS, ufmin_pred_ty> m_UnordFMin(const LHS &L,
+                                                          const RHS &R) {
+  return FMaxMin_match<LHS, RHS, ufmin_pred_ty>(L, R);
 }
 
 /// Match an 'ordered' or 'unordered' floating point maximum function.
@@ -2651,11 +2581,11 @@ m_UnordFMin(const LHS &L, const RHS &R) {
 /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'maximum'
 /// semantics.
 template <typename LHS, typename RHS>
-inline match_combine_or<MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty>,
-                        MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty>>
+inline match_combine_or<FMaxMin_match<LHS, RHS, ofmax_pred_ty>,
+                        FMaxMin_match<LHS, RHS, ufmax_pred_ty>>
 m_OrdOrUnordFMax(const LHS &L, const RHS &R) {
-  return m_CombineOr(MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty>(L, R),
-                     MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty>(L, R));
+  return m_CombineOr(FMaxMin_match<LHS, RHS, ofmax_pred_ty>(L, R),
+                     FMaxMin_match<LHS, RHS, ufmax_pred_ty>(L, R));
 }
 
 /// Match an 'ordered' or 'unordered' floating point minimum function.
@@ -2664,11 +2594,11 @@ m_OrdOrUnordFMax(const LHS &L, const RHS &R) {
 /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'minimum'
 /// semantics.
 template <typename LHS, typename RHS>
-inline match_combine_or<MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty>,
-                        MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty>>
+inline match_combine_or<FMaxMin_match<LHS, RHS, ofmin_pred_ty>,
+                        FMaxMin_match<LHS, RHS, ufmin_pred_ty>>
 m_OrdOrUnordFMin(const LHS &L, const RHS &R) {
-  return m_CombineOr(MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty>(L, R),
-                     MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty>(L, R));
+  return m_CombineOr(FMaxMin_match<LHS, RHS, ofmin_pred_ty>(L, R),
+                     FMaxMin_match<LHS, RHS, ufmin_pred_ty>(L, R));
 }
 
 /// Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
@@ -2962,6 +2892,32 @@ inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_Cttz(const Opnd0 &Op0,
   return m_Intrinsic<Intrinsic::cttz>(Op0, Op1);
 }
 
+template <typename Opnd0, typename Opnd1>
+inline auto m_SMax(const Opnd0 &Op0, const Opnd1 &Op1) {
+  return m_Intrinsic<Intrinsic::smax>(Op0, Op1);
+}
+
+template <typename Opnd0, typename Opnd1>
+inline auto m_SMin(const Opnd0 &Op0, const Opnd1 &Op1) {
+  return m_Intrinsic<Intrinsic::smin>(Op0, Op1);
+}
+
+template <typename Opnd0, typename Opnd1>
+inline auto m_UMax(const Opnd0 &Op0, const Opnd1 &Op1) {
+  return m_Intrinsic<Intrinsic::umax>(Op0, Op1);
+}
+
+template <typename Opnd0, typename Opnd1>
+inline auto m_UMin(const Opnd0 &Op0, const Opnd1 &Op1) {
+  return m_Intrinsic<Intrinsic::umin>(Op0, Op1);
+}
+
+template <typename Opnd0, typename Opnd1>
+inline auto m_MaxOrMin(const Opnd0 &Op0, const Opnd1 &Op1) {
+  return m_CombineOr(m_SMax(Op0, Op1), m_SMin(Op0, Op1), m_UMax(Op0, Op1),
+                     m_UMin(Op0, Op1));
+}
+
 template <typename Opnd0, typename Opnd1>
 inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMinNum(const Opnd0 &Op0,
                                                            const Opnd1 &Op1) {
@@ -3128,37 +3084,6 @@ m_NSWNeg(const ValTy &V) {
   return m_NSWSub(m_ZeroInt(), V);
 }
 
-/// Matches an SMin with LHS and RHS in either order.
-template <typename LHS, typename RHS>
-inline MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>
-m_c_SMin(const LHS &L, const RHS &R) {
-  return MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>(L, R);
-}
-/// Matches an SMax with LHS and RHS in either order.
-template <typename LHS, typename RHS>
-inline MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>
-m_c_SMax(const LHS &L, const RHS &R) {
-  return MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>(L, R);
-}
-/// Matches a UMin with LHS and RHS in either order.
-template <typename LHS, typename RHS>
-inline MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>
-m_c_UMin(const LHS &L, const RHS &R) {
-  return MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>(L, R);
-}
-/// Matches a UMax with LHS and RHS in either order.
-template <typename LHS, typename RHS>
-inline MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>
-m_c_UMax(const LHS &L, const RHS &R) {
-  return MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>(L, R);
-}
-
-template <typename LHS, typename RHS>
-inline auto m_c_MaxOrMin(const LHS &L, const RHS &R) {
-  return m_CombineOr(m_c_SMax(L, R), m_c_SMin(L, R), m_c_UMax(L, R),
-                     m_c_UMin(L, R));
-}
-
 template <Intrinsic::ID IntrID, typename LHS, typename RHS>
 struct CommutativeBinaryIntrinsic_match {
   LHS L;
@@ -3181,6 +3106,33 @@ m_c_Intrinsic(const T0 &Op0, const T1 &Op1) {
   return CommutativeBinaryIntrinsic_match<IntrID, T0, T1>(Op0, Op1);
 }
 
+/// Matches an SMin with LHS and RHS in either order.
+template <typename LHS, typename RHS>
+inline auto m_c_SMin(const LHS &L, const RHS &R) {
+  return m_c_Intrinsic<Intrinsic::smin>(L, R);
+}
+/// Matches an SMax with LHS and RHS in either order.
+template <typename LHS, typename RHS>
+inline auto m_c_SMax(const LHS &L, const RHS &R) {
+  return m_c_Intrinsic<Intrinsic::smax>(L, R);
+}
+/// Matches a UMin with LHS and RHS in either order.
+template <typename LHS, typename RHS>
+inline auto m_c_UMin(const LHS &L, const RHS &R) {
+  return m_c_Intrinsic<Intrinsic::umin>(L, R);
+}
+/// Matches a UMax with LHS and RHS in either order.
+template <typename LHS, typename RHS>
+inline auto m_c_UMax(const LHS &L, const RHS &R) {
+  return m_c_Intrinsic<Intrinsic::umax>(L, R);
+}
+
+template <typename LHS, typename RHS>
+inline auto m_c_MaxOrMin(const LHS &L, const RHS &R) {
+  return m_CombineOr(m_c_SMax(L, R), m_c_SMin(L, R), m_c_UMax(L, R),
+                     m_c_UMin(L, R));
+}
+
 /// Matches FAdd with LHS and RHS in either order.
 template <typename LHS, typename RHS>
 inline BinaryOp_match<LHS, RHS, Instruction::FAdd, true>
diff --git a/llvm/include/llvm/Support/CHERICapabilityFormat.h b/llvm/include/llvm/Support/CHERICapabilityFormat.h
index 616bbaff09666..bbf589df73339 100644
--- a/llvm/include/llvm/Support/CHERICapabilityFormat.h
+++ b/llvm/include/llvm/Support/CHERICapabilityFormat.h
@@ -23,21 +23,32 @@ struct CHERICapabilityFormatBase {
   /// Returns the "alignment mask" for an allocation of size \p Length. This
   /// mask is 0 where the capability format alignment requires the
   /// address to be 0, and 1 otherwise.
-  LLVM_ABI static AddressType getAlignmentMask(AddressType Length);
+  static AddressType getAlignmentMask(AddressType Length) {
+    return Derived::getAlignmentMaskImpl(Length);
+  }
 
   /// Returns the required alignment for an allocation of size \p Length.
-  LLVM_ABI static Align getRequiredAlignment(AddressType Length);
+  static Align getRequiredAlignment(AddressType Length) {
+    return Align((~getAlignmentMask(Length) + 1) & AddressMask);
+  }
 
   /// Returns \p Length rounded up to the nearest representable allocation
   /// length.
-  LLVM_ABI static AddressType getRepresentableLength(AddressType Length);
+  static AddressType getRepresentableLength(AddressType Length) {
+    AddressType Mask = getAlignmentMask(Length);
+    return (Length + ~Mask) & Mask;
+  }
 };
 
 template <typename AddressType, unsigned MW, unsigned MAX_E>
 struct RVYCapabilityFormat
     : public CHERICapabilityFormatBase<
           RVYCapabilityFormat<AddressType, MW, MAX_E>, AddressType> {
-  LLVM_ABI static AddressType getAlignmentMask(uint64_t Length);
+  friend struct CHERICapabilityFormatBase<
+      RVYCapabilityFormat<AddressType, MW, MAX_E>, AddressType>;
+
+private:
+  LLVM_ABI static AddressType getAlignmentMaskImpl(uint64_t Length);
 };
 
 using RV32YCapabilityFormat = RVYCapabilityFormat<uint32_t, 10, 24>;
@@ -45,7 +56,10 @@ using RV64YCapabilityFormat = RVYCapabilityFormat<uint64_t, 14, 52>;
 
 struct CHERIoTCapabilityFormat
     : public CHERICapabilityFormatBase<CHERIoTCapabilityFormat, uint32_t> {
-  LLVM_ABI static uint32_t getAlignmentMask(uint32_t Length);
+  friend struct CHERICapabilityFormatBase<CHERIoTCapabilityFormat, uint32_t>;
+
+private:
+  LLVM_ABI static uint32_t getAlignmentMaskImpl(uint32_t Length);
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Transforms/Scalar/NaryReassociate.h b/llvm/include/llvm/Transforms/Scalar/NaryReassociate.h
index 7d0840f89c382..fa41ed1473c29 100644
--- a/llvm/include/llvm/Transforms/Scalar/NaryReassociate.h
+++ b/llvm/include/llvm/Transforms/Scalar/NaryReassociate.h
@@ -93,6 +93,7 @@ class DominatorTree;
 class Function;
 class GetElementPtrInst;
 class Instruction;
+class IntrinsicInst;
 class TargetLibraryInfo;
 class TargetTransformInfo;
 class Type;
@@ -156,17 +157,8 @@ class NaryReassociatePass : public OptionalPassInfoMixin<NaryReassociatePass> {
   Instruction *findClosestMatchingDominator(SCEVUse CandidateExpr,
                                             Instruction *Dominatee);
 
-  // Try to match \p I as signed/unsigned Min/Max and reassociate it. \p
-  // OrigSCEV is set if \I matches Min/Max regardless whether resassociation is
-  // done or not. If reassociation was successful newly generated instruction is
-  // returned, otherwise nullptr.
-  template <typename PredT>
-  Instruction *matchAndReassociateMinOrMax(Instruction *I, SCEVUse &OrigSCEV);
-
   // Reassociate Min/Max.
-  template <typename MaxMinT>
-  Value *tryReassociateMinOrMax(Instruction *I, MaxMinT MaxMinMatch, Value *LHS,
-                                Value *RHS);
+  Value *tryReassociateMinOrMax(IntrinsicInst *I);
 
   // GetElementPtrInst implicitly sign-extends an index if the index is shorter
   // than the pointer size. This function returns whether Index is shorter than
diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp
index 55d5569dc6ba0..a2e321884e708 100644
--- a/llvm/lib/Analysis/InlineCost.cpp
+++ b/llvm/lib/Analysis/InlineCost.cpp
@@ -3245,7 +3245,7 @@ std::optional<InlineResult> llvm::getAttributeBasedInliningDecision(
     return InlineResult::failure("optnone attribute");
 
   // Don't inline functions which can be interposed at link-time.
-  if (Callee->isInterposable())
+  if (Callee->isInterposable(/*CheckNoIPA=*/false))
     return InlineResult::failure("interposable");
 
   // Don't inline functions marked noinline.
diff --git a/llvm/lib/Analysis/ValueLatticeUtils.cpp b/llvm/lib/Analysis/ValueLatticeUtils.cpp
index 2bcb4d5b0e6b1..acc19667d7764 100644
--- a/llvm/lib/Analysis/ValueLatticeUtils.cpp
+++ b/llvm/lib/Analysis/ValueLatticeUtils.cpp
@@ -17,7 +17,8 @@
 using namespace llvm;
 
 bool llvm::canTrackArgumentsInterprocedurally(Function *F) {
-  return F->hasLocalLinkage() && !F->hasAddressTaken();
+  return F->hasExactDefinition() && F->hasLocalLinkage() &&
+         !F->hasAddressTaken();
 }
 
 bool llvm::canTrackReturnsInterprocedurally(Function *F) {
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index f20707c3e7692..c79dcfb05da10 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -2310,6 +2310,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
     return Attribute::DenormalFPEnv;
   case bitc::ATTR_KIND_NOOUTLINE:
     return Attribute::NoOutline;
+  case bitc::ATTR_KIND_NOIPA:
+    return Attribute::NoIPA;
   }
 }
 
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index c25232037194b..456ebafa9b256 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -1015,6 +1015,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
     return bitc::ATTR_KIND_DENORMAL_FPENV;
   case Attribute::NoOutline:
     return bitc::ATTR_KIND_NOOUTLINE;
+  case Attribute::NoIPA:
+    return bitc::ATTR_KIND_NOIPA;
   case Attribute::EndAttrKinds:
     llvm_unreachable("Can not encode end-attribute kinds marker.");
   case Attribute::None:
diff --git a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
index 60eb96fc5de1b..60a660557a0f3 100644
--- a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
+++ b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
@@ -1245,11 +1245,8 @@ ComplexDeinterleavingGraph::identifyReassocNodes(Instruction *Real,
   auto Collect = [&Flags](Instruction *Insn, SmallVectorImpl<Product> &Muls,
                           AddendList &Addends) -> bool {
     SmallVector<PointerIntPair<Value *, 1, bool>> Worklist = {{Insn, true}};
-    SmallPtrSet<Value *, 8> Visited;
     while (!Worklist.empty()) {
       auto [V, IsPositive] = Worklist.pop_back_val();
-      if (!Visited.insert(V).second)
-        continue;
 
       Instruction *I = dyn_cast<Instruction>(V);
       if (!I) {
diff --git a/llvm/lib/IR/Globals.cpp b/llvm/lib/IR/Globals.cpp
index 805595a80a054..9685231d3ea32 100644
--- a/llvm/lib/IR/Globals.cpp
+++ b/llvm/lib/IR/Globals.cpp
@@ -113,7 +113,9 @@ GlobalObject::~GlobalObject() {
   setComdat(nullptr);
 }
 
-bool GlobalValue::isInterposable() const {
+bool GlobalValue::isInterposable(bool CheckNoIPA) const {
+  if (CheckNoIPA && isNoipaFnDef())
+    return true;
   if (isInterposableLinkage(getLinkage()))
     return true;
   return !isDSOLocal() && getParent() &&
@@ -334,6 +336,13 @@ bool GlobalValue::isNobuiltinFnDef() const {
   return F->hasFnAttribute(Attribute::NoBuiltin);
 }
 
+bool GlobalValue::isNoipaFnDef() const {
+  const Function *F = dyn_cast<Function>(this);
+  if (!F || F->isDeclaration())
+    return false;
+  return F->hasFnAttribute(Attribute::NoIPA);
+}
+
 bool GlobalValue::isDeclaration() const {
   // Globals are definitions if they have an initializer.
   if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(this))
diff --git a/llvm/lib/Support/CHERICapabilityFormat.cpp b/llvm/lib/Support/CHERICapabilityFormat.cpp
index bc2ba5ddeca36..322870affc2f1 100644
--- a/llvm/lib/Support/CHERICapabilityFormat.cpp
+++ b/llvm/lib/Support/CHERICapabilityFormat.cpp
@@ -11,23 +11,9 @@
 
 namespace llvm {
 
-template <typename Derived, typename AddressType>
-Align CHERICapabilityFormatBase<Derived, AddressType>::getRequiredAlignment(
-    AddressType Length) {
-  return Align((~Derived::getAlignmentMask(Length) + 1) & AddressMask);
-}
-
-template <typename Derived, typename AddressType>
-AddressType
-CHERICapabilityFormatBase<Derived, AddressType>::getRepresentableLength(
-    AddressType Length) {
-  AddressType Mask = Derived::getAlignmentMask(Length);
-  return (Length + ~Mask) & Mask;
-}
-
 template <typename AddressType, unsigned MW, unsigned MAX_E>
-AddressType
-RVYCapabilityFormat<AddressType, MW, MAX_E>::getAlignmentMask(uint64_t Length) {
+AddressType RVYCapabilityFormat<AddressType, MW, MAX_E>::getAlignmentMaskImpl(
+    uint64_t Length) {
   static constexpr unsigned int IE_TAKE_BITS = 3;
 
   if (Length == 0)
@@ -57,7 +43,7 @@ template struct CHERICapabilityFormatBase<RVYCapabilityFormat<uint64_t, 14, 52>,
 template struct RVYCapabilityFormat<uint32_t, 10, 24>;
 template struct RVYCapabilityFormat<uint64_t, 14, 52>;
 
-uint32_t CHERIoTCapabilityFormat::getAlignmentMask(uint32_t Length) {
+uint32_t CHERIoTCapabilityFormat::getAlignmentMaskImpl(uint32_t Length) {
   // Per section 7.13.4 and table 7.4 in the v1.0 CHERIoT specification.
   constexpr uint32_t NINE_SET_BITS = 511;
   uint32_t E;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 448a983270602..730112f95084f 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -19758,7 +19758,7 @@ bool AArch64TargetLowering::isFMAFasterThanFMulAndFAdd(
     return Subtarget->hasFullFP16();
   case MVT::f32:
   case MVT::f64:
-    return true;
+    return Subtarget->hasFPARMv8();
   case MVT::bf16:
     return VT.isScalableVector() && Subtarget->hasBF16() &&
            Subtarget->isNonStreamingSVEorSME2Available();
@@ -19774,7 +19774,7 @@ bool AArch64TargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F,
   switch (Ty->getScalarType()->getTypeID()) {
   case Type::FloatTyID:
   case Type::DoubleTyID:
-    return true;
+    return Subtarget->hasFPARMv8();
   default:
     return false;
   }
@@ -33080,7 +33080,8 @@ SDValue AArch64TargetLowering::LowerToScalableOp(SDValue Op,
     Ops.push_back(convertToScalableVector(DAG, ContainerVT, V));
   }
 
-  auto ScalableRes = DAG.getNode(Op.getOpcode(), SDLoc(Op), ContainerVT, Ops);
+  auto ScalableRes =
+      DAG.getNode(Op.getOpcode(), SDLoc(Op), ContainerVT, Ops, Op->getFlags());
   return convertFromScalableVector(DAG, VT, ScalableRes);
 }
 
diff --git a/llvm/lib/Target/AArch64/AArch64SchedA510.td b/llvm/lib/Target/AArch64/AArch64SchedA510.td
index 8a13067ca197b..56ac7141be30f 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedA510.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedA510.td
@@ -124,18 +124,16 @@ def : WriteRes<WriteLD, [CortexA510UnitLd]> { let Latency = 2; }
 def : WriteRes<WriteLDIdx, [CortexA510UnitLd]> { let Latency = 2; }
 def : WriteRes<WriteLDHi, [CortexA510UnitLd]> { let Latency = 2; }
 
+def CortexA510Write_3c_1Ld : SchedWriteRes<[CortexA510UnitLd]> { let Latency = 3; }
+def CortexA510Write_3c_2Ld : SchedWriteRes<[CortexA510UnitLd, CortexA510UnitLd]> { let Latency = 3; }
+def CortexA510Write_4c_2Ld : SchedWriteRes<[CortexA510UnitLd, CortexA510UnitLd]> { let Latency = 4; }
+def CortexA510Write_4c_2r_2Ld : SchedWriteRes<[CortexA510UnitLd, CortexA510UnitLd]> { let Latency = 4; let ReleaseAtCycles = [2,2]; }
+def CortexA510Write_5c_3r_2Ld : SchedWriteRes<[CortexA510UnitLd, CortexA510UnitLd]> { let Latency = 5; let ReleaseAtCycles = [3,3]; }
+def CortexA510Write_5c_4r_2Ld : SchedWriteRes<[CortexA510UnitLd, CortexA510UnitLd]> { let Latency = 5; let ReleaseAtCycles = [4,4]; }
+def CortexA510Write_6c_4r_2Ld : SchedWriteRes<[CortexA510UnitLd, CortexA510UnitLd]> { let Latency = 6; let ReleaseAtCycles = [4,4]; }
+
 def CortexA510WriteVLD1 : SchedWriteRes<[CortexA510UnitLd]> { let Latency = 3; }
 def CortexA510WriteVLD1SI : SchedWriteRes<[CortexA510UnitLd]> { let Latency = 3; let SingleIssue = 1; }
-def CortexA510WriteVLD2 : SchedWriteRes<[CortexA510UnitLd]> { let Latency = 4;
-                                                  let ReleaseAtCycles = [2]; }
-def CortexA510WriteVLD3 : SchedWriteRes<[CortexA510UnitLd]> { let Latency = 5;
-                                                  let ReleaseAtCycles = [3]; }
-def CortexA510WriteVLD4 : SchedWriteRes<[CortexA510UnitLd]> { let Latency = 6;
-                                                  let ReleaseAtCycles = [4]; }
-def CortexA510WriteVLD6 : SchedWriteRes<[CortexA510UnitLd]> { let Latency = 5;
-                                                  let ReleaseAtCycles = [3]; }
-def CortexA510WriteVLD8 : SchedWriteRes<[CortexA510UnitLd]> { let Latency = 6;
-                                                  let ReleaseAtCycles = [4]; }
 
 def CortexA510WriteLDP1 : SchedWriteRes<[CortexA510UnitLd]> { let Latency = 3; }
 def CortexA510WriteLDP2 : SchedWriteRes<[CortexA510UnitLd]> { let Latency = 3; }
@@ -284,60 +282,64 @@ def : InstRW<[WriteI], (instrs COPY)>;
 // Vector Loads - 128-bit per cycle
 //---
 //   1-element structures
-def : InstRW<[CortexA510WriteVLD1], (instregex "LD1i(8|16|32|64)$")>;                // single element
-def : InstRW<[CortexA510WriteVLD1], (instregex "LD1Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>; // replicate
-def : InstRW<[CortexA510WriteVLD1], (instregex "LD1Onev(8b|4h|2s|1d)$")>;
-def : InstRW<[CortexA510WriteVLD1], (instregex "LD1Onev(16b|8h|4s|2d)$")>;
-def : InstRW<[CortexA510WriteVLD1], (instregex "LD1Twov(8b|4h|2s|1d)$")>; // multiple structures
-def : InstRW<[CortexA510WriteVLD1], (instregex "LD1Twov(16b|8h|4s|2d)$")>;
-def : InstRW<[CortexA510WriteVLD2], (instregex "LD1Threev(8b|4h|2s|1d)$")>;
-def : InstRW<[CortexA510WriteVLD2], (instregex "LD1Threev(16b|8h|4s|2d)$")>;
-def : InstRW<[CortexA510WriteVLD2], (instregex "LD1Fourv(8b|4h|2s|1d)$")>;
-def : InstRW<[CortexA510WriteVLD2], (instregex "LD1Fourv(16b|8h|4s|2d)$")>;
-
-def : InstRW<[WriteAdr, CortexA510WriteVLD1], (instregex "LD1i(8|16|32|64)_POST$")>;
-def : InstRW<[WriteAdr, CortexA510WriteVLD1], (instregex "LD1Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
-def : InstRW<[WriteAdr, CortexA510WriteVLD1], (instregex "LD1Onev(8b|4h|2s|1d)_POST$")>;
-def : InstRW<[WriteAdr, CortexA510WriteVLD2], (instregex "LD1Onev(16b|8h|4s|2d)_POST$")>;
-def : InstRW<[WriteAdr, CortexA510WriteVLD2], (instregex "LD1Twov(8b|4h|2s|1d)_POST$")>;
-def : InstRW<[WriteAdr, CortexA510WriteVLD2], (instregex "LD1Twov(16b|8h|4s|2d)_POST$")>;
-def : InstRW<[WriteAdr, CortexA510WriteVLD2], (instregex "LD1Threev(8b|4h|2s|1d)_POST$")>;
-def : InstRW<[WriteAdr, CortexA510WriteVLD2], (instregex "LD1Threev(16b|8h|4s|2d)_POST$")>;
-def : InstRW<[WriteAdr, CortexA510WriteVLD2], (instregex "LD1Fourv(8b|4h|2s|1d)_POST$")>;
-def : InstRW<[WriteAdr, CortexA510WriteVLD2], (instregex "LD1Fourv(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[CortexA510Write_3c_1Ld], (instregex "LD1Onev(8b|4h|2s|1d)$")>;       // 1 element, multiple, 1 reg, D-form
+def : InstRW<[CortexA510Write_3c_1Ld], (instregex "LD1Onev(16b|8h|4s|2d)$")>;      // 1 element, multiple, 1 reg, Q-form
+def : InstRW<[CortexA510Write_3c_2Ld], (instregex "LD1Twov(8b|4h|2s|1d)$")>;       // 1 element, multiple, 2 reg, D-form
+def : InstRW<[CortexA510Write_3c_2Ld], (instregex "LD1Twov(16b|8h|4s|2d)$")>;      // 1 element, multiple, 2 reg, Q-form
+def : InstRW<[CortexA510Write_4c_2r_2Ld], (instregex "LD1Threev(8b|4h|2s|1d)$")>;  // 1 element, multiple, 3 reg, D-form
+def : InstRW<[CortexA510Write_4c_2r_2Ld], (instregex "LD1Threev(16b|8h|4s|2d)$")>; // 1 element, multiple, 3 reg, Q-form
+def : InstRW<[CortexA510Write_4c_2r_2Ld], (instregex "LD1Fourv(8b|4h|2s|1d)$")>;   // 1 element, multiple, 4 reg, D-form
+def : InstRW<[CortexA510Write_4c_2r_2Ld], (instregex "LD1Fourv(16b|8h|4s|2d)$")>;  // 1 element, multiple, 4 reg, Q-form
+def : InstRW<[CortexA510Write_3c_1Ld], (instregex "LD1i(8|16|32|64)$")>;                // 1 element, one lane
+def : InstRW<[CortexA510Write_3c_1Ld], (instregex "LD1Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>; // 1 element, all lanes
+
+def : InstRW<[WriteAdr, CortexA510Write_3c_1Ld], (instregex "LD1Onev(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA510Write_3c_1Ld], (instregex "LD1Onev(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA510Write_3c_2Ld], (instregex "LD1Twov(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA510Write_3c_2Ld], (instregex "LD1Twov(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA510Write_4c_2r_2Ld], (instregex "LD1Threev(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA510Write_4c_2r_2Ld], (instregex "LD1Threev(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA510Write_4c_2r_2Ld], (instregex "LD1Fourv(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA510Write_4c_2r_2Ld], (instregex "LD1Fourv(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA510Write_3c_1Ld], (instregex "LD1i(8|16|32|64)_POST$")>;
+def : InstRW<[WriteAdr, CortexA510Write_3c_1Ld], (instregex "LD1Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 
 //    2-element structures
-def : InstRW<[CortexA510WriteVLD2], (instregex "LD2i(8|16|32|64)$")>;
-def : InstRW<[CortexA510WriteVLD2], (instregex "LD2Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
-def : InstRW<[CortexA510WriteVLD2], (instregex "LD2Twov(8b|4h|2s)$")>;
-def : InstRW<[CortexA510WriteVLD4], (instregex "LD2Twov(16b|8h|4s|2d)$")>;
-
-def : InstRW<[WriteAdr, CortexA510WriteVLD2], (instregex "LD2i(8|16|32|64)(_POST)?$")>;
-def : InstRW<[WriteAdr, CortexA510WriteVLD2], (instregex "LD2Rv(8b|4h|2s|1d|16b|8h|4s|2d)(_POST)?$")>;
-def : InstRW<[WriteAdr, CortexA510WriteVLD2], (instregex "LD2Twov(8b|4h|2s)(_POST)?$")>;
-def : InstRW<[WriteAdr, CortexA510WriteVLD4], (instregex "LD2Twov(16b|8h|4s|2d)(_POST)?$")>;
+def : InstRW<[CortexA510Write_4c_2Ld], (instregex "LD2Twov(8b|4h|2s)$")>;     // 2 element, multiple, D-form
+def : InstRW<[CortexA510Write_4c_2r_2Ld], (instregex "LD2Twov(16b|8h|4s)$")>; // 2 element, multiple, Q-form, B/H/S
+def : InstRW<[CortexA510Write_4c_2Ld], (instregex "LD2Twov(2d)$")>;           // 2 element, multiple, Q-form, D
+def : InstRW<[CortexA510Write_4c_2r_2Ld], (instregex "LD2i(8|16|32|64)$")>;             // 2 element, one lane
+def : InstRW<[CortexA510Write_3c_2Ld], (instregex "LD2Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>; // 2 element, all lanes
+
+def : InstRW<[WriteAdr, CortexA510Write_4c_2Ld], (instregex "LD2Twov(8b|4h|2s)_POST$")>;
+def : InstRW<[WriteAdr, CortexA510Write_4c_2r_2Ld], (instregex "LD2Twov(16b|8h|4s)_POST$")>;
+def : InstRW<[WriteAdr, CortexA510Write_4c_2Ld], (instregex "LD2Twov(2d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA510Write_4c_2r_2Ld], (instregex "LD2i(8|16|32|64)_POST$")>;
+def : InstRW<[WriteAdr, CortexA510Write_3c_2Ld], (instregex "LD2Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 
 //    3-element structures
-def : InstRW<[CortexA510WriteVLD2], (instregex "LD3i(8|16|32|64)$")>;
-def : InstRW<[CortexA510WriteVLD2], (instregex "LD3Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
-def : InstRW<[CortexA510WriteVLD3], (instregex "LD3Threev(8b|4h|2s|1d)$")>;
-def : InstRW<[CortexA510WriteVLD6], (instregex "LD3Threev(16b|8h|4s|2d)$")>;
+def : InstRW<[CortexA510Write_5c_3r_2Ld], (instregex "LD3Threev(8b|4h|2s|1d)$")>;  // 3 element, multiple, D-form
+def : InstRW<[CortexA510Write_5c_3r_2Ld], (instregex "LD3Threev(16b|8h|4s|2d)$")>; // 3 element, multiple, Q-form
+def : InstRW<[CortexA510Write_5c_3r_2Ld], (instregex "LD3i(8|16|32|64)$")>;                // 3 element, one lane
+def : InstRW<[CortexA510Write_4c_2r_2Ld], (instregex "LD3Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>; // 3 element, all lanes
 
-def : InstRW<[WriteAdr, CortexA510WriteVLD2], (instregex "LD3i(8|16|32|64)_POST$")>;
-def : InstRW<[WriteAdr, CortexA510WriteVLD2], (instregex "LD3Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
-def : InstRW<[WriteAdr, CortexA510WriteVLD3], (instregex "LD3Threev(8b|4h|2s|1d)_POST$")>;
-def : InstRW<[WriteAdr, CortexA510WriteVLD6], (instregex "LD3Threev(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA510Write_5c_3r_2Ld], (instregex "LD3Threev(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA510Write_5c_3r_2Ld], (instregex "LD3Threev(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA510Write_5c_3r_2Ld], (instregex "LD3i(8|16|32|64)_POST$")>;
+def : InstRW<[WriteAdr, CortexA510Write_4c_2r_2Ld], (instregex "LD3Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 
 //    4-element structures
-def : InstRW<[CortexA510WriteVLD2], (instregex "LD4i(8|16|32|64)$")>;                // load single 4-el structure to one lane of 4 regs.
-def : InstRW<[CortexA510WriteVLD2], (instregex "LD4Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>; // load single 4-el structure, replicate to all lanes of 4 regs.
-def : InstRW<[CortexA510WriteVLD4], (instregex "LD4Fourv(8b|4h|2s|1d)$")>;           // load multiple 4-el structures to 4 regs.
-def : InstRW<[CortexA510WriteVLD8], (instregex "LD4Fourv(16b|8h|4s|2d)$")>;
-
-def : InstRW<[WriteAdr, CortexA510WriteVLD2], (instregex "LD4i(8|16|32|64)_POST$")>;
-def : InstRW<[WriteAdr, CortexA510WriteVLD2], (instregex "LD4Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
-def : InstRW<[WriteAdr, CortexA510WriteVLD4], (instregex "LD4Fourv(8b|4h|2s|1d)_POST$")>;
-def : InstRW<[WriteAdr, CortexA510WriteVLD8], (instregex "LD4Fourv(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[CortexA510Write_5c_3r_2Ld], (instregex "LD4Fourv(8b|4h|2s|1d)$")>; // 4 element, multiple, D-form
+def : InstRW<[CortexA510Write_5c_3r_2Ld], (instregex "LD4Fourv(16b|8h|4s)$")>;   // 4 element, multiple, D-form, B/H/S
+def : InstRW<[CortexA510Write_5c_4r_2Ld], (instregex "LD4Fourv(2d)$")>;          // 4 element, multiple, D-form, D
+def : InstRW<[CortexA510Write_6c_4r_2Ld], (instregex "LD4i(8|16|32|64)$")>;                // 4 element, one lane
+def : InstRW<[CortexA510Write_4c_2r_2Ld], (instregex "LD4Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>; // 4 element, all lanes
+
+def : InstRW<[WriteAdr, CortexA510Write_5c_3r_2Ld], (instregex "LD4Fourv(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA510Write_5c_3r_2Ld], (instregex "LD4Fourv(16b|8h|4s)_POST$")>;
+def : InstRW<[WriteAdr, CortexA510Write_5c_4r_2Ld], (instregex "LD4Fourv(2d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA510Write_6c_4r_2Ld], (instregex "LD4i(8|16|32|64)_POST$")>;
+def : InstRW<[WriteAdr, CortexA510Write_4c_2r_2Ld], (instregex "LD4Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 
 //---
 // Vector Stores
diff --git a/llvm/lib/Target/NVPTX/CMakeLists.txt b/llvm/lib/Target/NVPTX/CMakeLists.txt
index e0e2b1a40bffc..d2b5083a5f83f 100644
--- a/llvm/lib/Target/NVPTX/CMakeLists.txt
+++ b/llvm/lib/Target/NVPTX/CMakeLists.txt
@@ -12,6 +12,7 @@ tablegen(LLVM NVPTXGenSubtargetInfo.inc -gen-subtarget)
 add_public_tablegen_target(NVPTXCommonTableGen)
 
 set(NVPTXCodeGen_sources
+  NVPTXAddressFolder.cpp
   NVPTXAliasAnalysis.cpp
   NVPTXAllocaHoisting.cpp
   NVPTXAsmPrinter.cpp
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp
index 5933f74fcb25f..ef447e4ceed6b 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp
@@ -24,6 +24,7 @@
 using namespace llvm;
 
 #define GET_INSTRINFO_MC_DESC
+#define GET_INSTRINFO_NAMED_OPS
 #define ENABLE_INSTR_PREDICATE_VERIFIER
 #include "NVPTXGenInstrInfo.inc"
 
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.h b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.h
index 78f4e67455023..58083e1248bb0 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.h
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_LIB_TARGET_NVPTX_MCTARGETDESC_NVPTXMCTARGETDESC_H
 #define LLVM_LIB_TARGET_NVPTX_MCTARGETDESC_NVPTXMCTARGETDESC_H
 
+#include "llvm/Support/Compiler.h"
 #include <stdint.h>
 
 // Defines symbolic names for PTX registers.
@@ -22,6 +23,7 @@
 // Defines symbolic names for the PTX instructions.
 #define GET_INSTRINFO_ENUM
 #define GET_INSTRINFO_MC_HELPER_DECLS
+#define GET_INSTRINFO_OPERAND_ENUM
 #include "NVPTXGenInstrInfo.inc"
 
 #define GET_SUBTARGETINFO_ENUM
diff --git a/llvm/lib/Target/NVPTX/NVPTX.h b/llvm/lib/Target/NVPTX/NVPTX.h
index 08ccc143dfb8a..57ed9bb43aeea 100644
--- a/llvm/lib/Target/NVPTX/NVPTX.h
+++ b/llvm/lib/Target/NVPTX/NVPTX.h
@@ -59,6 +59,7 @@ FunctionPass *createNVPTXIRPeepholePass();
 MachineFunctionPass *createNVPTXPeephole();
 MachineFunctionPass *createNVPTXProxyRegErasurePass();
 MachineFunctionPass *createNVPTXForwardParamsPass();
+MachineFunctionPass *createNVPTXAddressFolderPass();
 
 void initializeNVVMReflectLegacyPassPass(PassRegistry &);
 void initializeGenericToNVVMLegacyPassPass(PassRegistry &);
@@ -74,6 +75,7 @@ void initializeNVPTXLowerArgsLegacyPassPass(PassRegistry &);
 void initializeNVPTXSetByValParamAlignLegacyPassPass(PassRegistry &);
 void initializeNVPTXProxyRegErasurePass(PassRegistry &);
 void initializeNVPTXForwardParamsPassPass(PassRegistry &);
+void initializeNVPTXAddressFolderPassPass(PassRegistry &);
 void initializeNVVMIntrRangePass(PassRegistry &);
 void initializeNVVMReflectPass(PassRegistry &);
 void initializeNVPTXAAWrapperPassPass(PassRegistry &);
@@ -294,6 +296,7 @@ void initializeNVPTXDAGToDAGISelLegacyPass(PassRegistry &);
 // Defines symbolic names for the NVPTX instructions.
 #define GET_INSTRINFO_ENUM
 #define GET_INSTRINFO_MC_HELPER_DECLS
+#define GET_INSTRINFO_OPERAND_ENUM
 #include "NVPTXGenInstrInfo.inc"
 
 #endif
diff --git a/llvm/lib/Target/NVPTX/NVPTXAddressFolder.cpp b/llvm/lib/Target/NVPTX/NVPTXAddressFolder.cpp
new file mode 100644
index 0000000000000..5299dbe686611
--- /dev/null
+++ b/llvm/lib/Target/NVPTX/NVPTXAddressFolder.cpp
@@ -0,0 +1,123 @@
+//===- NVPTXAddressFolder.cpp - Fold symbol addresses into memory ops -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// SelectionDAG folds a symbol address (a kernel parameter, global variable, or
+// external symbol) directly into the address operand of a memory access, but
+// only within a single basic block. When the address is carried across a block
+// boundary in a register it is materialized with a generic `mov`
+// (MOV_B{32,64}_sym) and the accesses become register-relative:
+//
+//     mov.b64       %rd1, kernel_param_0;
+//     ld.param.b64  %rd2, [%rd1];
+//     ld.param.b64  %rd3, [%rd1+8];
+//
+// This pass folds the symbol back into those address operands, eliminating the
+// redundant address arithmetic (and the `mov` itself once no use remains):
+//
+//     ld.param.b64  %rd2, [kernel_param_0];
+//     ld.param.b64  %rd3, [kernel_param_0+8];
+//
+// Shared-memory accesses are left alone: `mov`s of shared symbols are
+// deliberately kept CSE-able rather than duplicated into their uses, as
+// rematerializing them has caused performance regressions before (see
+// MovSymInst in NVPTXInstrInfo.td).
+//
+//===----------------------------------------------------------------------===//
+
+#include "NVPTX.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+
+using namespace llvm;
+
+// Try to fold the definition of \p Addr into \p MI's address operand. The
+// defining instruction is erased once it has no uses left; it is kept for any
+// remaining use, e.g. because the address also feeds arithmetic or escapes.
+static bool foldAddress(MachineInstr &MI, MachineOperand &Addr,
+                        MachineRegisterInfo &MRI) {
+  assert(Addr.isReg() && "Expected an address register");
+  assert(Addr.getReg().isVirtual() && "Expected a virtual address register");
+
+  MachineInstr *Mov = MRI.getVRegDef(Addr.getReg());
+  if (!Mov || (Mov->getOpcode() != NVPTX::MOV_B32_sym &&
+               Mov->getOpcode() != NVPTX::MOV_B64_sym))
+    return false;
+
+  const MachineOperand &Sym = Mov->getOperand(1);
+  if (!Sym.isGlobal() && !Sym.isSymbol())
+    return false;
+
+  // The accessed address space must be known and must not be shared.
+  const int AddrSpaceIdx =
+      NVPTX::getNamedOperandIdx(MI.getOpcode(), NVPTX::OpName::addsp);
+  if (AddrSpaceIdx < 0)
+    return false;
+  const auto AddrSpace = MI.getOperand(AddrSpaceIdx).getImm();
+  if (AddrSpace == NVPTX::AddressSpace::Shared ||
+      AddrSpace == NVPTX::AddressSpace::SharedCluster)
+    return false;
+
+  if (Sym.isGlobal()) {
+    Addr.ChangeToGA(Sym.getGlobal(), Sym.getOffset(), Sym.getTargetFlags());
+  } else {
+    Addr.ChangeToES(Sym.getSymbolName(), Sym.getTargetFlags());
+  }
+
+  if (MRI.use_empty(Mov->getOperand(0).getReg()))
+    Mov->eraseFromParent();
+
+  return true;
+}
+
+static bool foldAddresses(MachineFunction &MF) {
+  MachineRegisterInfo &MRI = MF.getRegInfo();
+
+  bool Changed = false;
+  for (MachineBasicBlock &MBB : MF)
+    for (MachineInstr &MI : make_early_inc_range(MBB))
+      if (MI.mayLoadOrStore()) {
+        const int AddrIdx =
+            NVPTX::getNamedOperandIdx(MI.getOpcode(), NVPTX::OpName::addr);
+        if (AddrIdx >= 0 && MI.getOperand(AddrIdx).isReg())
+          Changed |= foldAddress(MI, MI.getOperand(AddrIdx), MRI);
+      }
+
+  return Changed;
+}
+
+/// ----------------------------------------------------------------------------
+///                       Pass (Manager) Boilerplate
+/// ----------------------------------------------------------------------------
+
+namespace {
+struct NVPTXAddressFolderPass : public MachineFunctionPass {
+  static char ID;
+  NVPTXAddressFolderPass() : MachineFunctionPass(ID) {}
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+};
+} // namespace
+
+char NVPTXAddressFolderPass::ID = 0;
+
+INITIALIZE_PASS(NVPTXAddressFolderPass, "nvptx-address-folder",
+                "NVPTX Address Folder", false, false)
+
+bool NVPTXAddressFolderPass::runOnMachineFunction(MachineFunction &MF) {
+  return foldAddresses(MF);
+}
+
+MachineFunctionPass *llvm::createNVPTXAddressFolderPass() {
+  return new NVPTXAddressFolderPass();
+}
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrFormats.td b/llvm/lib/Target/NVPTX/NVPTXInstrFormats.td
index 719be0300940e..361f819a340ae 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrFormats.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrFormats.td
@@ -22,6 +22,7 @@ class NVPTXInst<dag outs, dag ins, string asmstr, list<dag> pattern = []>
   dag InOperandList = ins;
   let AsmString = asmstr;
   let Pattern = pattern;
+  let UseNamedOperandTable = true;
 
   // TSFlagFields
   bit IsLoad = false;
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index 5f1c16dc7f0fb..c9db5c5cec025 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -2454,8 +2454,8 @@ let isTerminator=1 in {
 
   let isBranch=1 in {
     def CBranch : NVPTXInst<(outs),
-                              (ins B1:$a, brtarget:$target, BranchFlag:$not),
-                              "@${not}$a bra \t$target;",
+                              (ins B1:$a, brtarget:$target, BranchFlag:$negate),
+                              "@${negate}$a bra \t$target;",
                               [(brcond i1:$a, bb:$target)]>;
 
     let isBarrier=1 in
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
index 6054feeedf932..2bc2158fccae1 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
@@ -119,6 +119,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeNVPTXTarget() {
   initializeNVPTXLowerAggrCopiesPass(PR);
   initializeNVPTXProxyRegErasurePass(PR);
   initializeNVPTXForwardParamsPassPass(PR);
+  initializeNVPTXAddressFolderPassPass(PR);
   initializeNVPTXDAGToDAGISelLegacyPass(PR);
   initializeNVPTXAAWrapperPassPass(PR);
   initializeNVPTXExternalAAWrapperPass(PR);
@@ -421,6 +422,8 @@ bool NVPTXPassConfig::addInstSelector() {
 
 void NVPTXPassConfig::addPreRegAlloc() {
   addPass(createNVPTXForwardParamsPass());
+  if (getOptLevel() != CodeGenOptLevel::None)
+    addPass(createNVPTXAddressFolderPass());
   // Remove Proxy Register pseudo instructions used to keep `callseq_end` alive.
   addPass(createNVPTXProxyRegErasurePass());
 }
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 5611410469384..df0a34ea27685 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -2076,7 +2076,9 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
   case RISCVISD::SUBD:
   case RISCVISD::PPAIRE_DB:
   case RISCVISD::WADDAU:
-  case RISCVISD::WSUBAU: {
+  case RISCVISD::WSUBAU:
+  case RISCVISD::WADDA:
+  case RISCVISD::WSUBA: {
     assert(!Subtarget->is64Bit() && "Unexpected opcode");
     assert(
         (Node->getOpcode() != RISCVISD::PPAIRE_DB || Subtarget->hasStdExtP()) &&
@@ -2096,10 +2098,27 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
     SDValue Op1Hi = Node->getOperand(3);
 
     MachineSDNode *New;
-    if (Opcode == RISCVISD::WADDAU || Opcode == RISCVISD::WSUBAU) {
-      // WADDAU/WSUBAU: Op0 is the accumulator (GPRPair), Op1Lo and Op1Hi are
-      // the two 32-bit values.
-      unsigned Opc = Opcode == RISCVISD::WADDAU ? RISCV::WADDAU : RISCV::WSUBAU;
+    if (Opcode == RISCVISD::WADDAU || Opcode == RISCVISD::WSUBAU ||
+        Opcode == RISCVISD::WADDA || Opcode == RISCVISD::WSUBA) {
+      // Widening accumulate: Op0 is the accumulator (GPRPair), Op1Lo and Op1Hi
+      // are the two 32-bit values.
+      unsigned Opc;
+      switch (Opcode) {
+      default:
+        llvm_unreachable("Unexpected opcode");
+      case RISCVISD::WADDAU:
+        Opc = RISCV::WADDAU;
+        break;
+      case RISCVISD::WSUBAU:
+        Opc = RISCV::WSUBAU;
+        break;
+      case RISCVISD::WADDA:
+        Opc = RISCV::WADDA;
+        break;
+      case RISCVISD::WSUBA:
+        Opc = RISCV::WSUBA;
+        break;
+      }
       New = CurDAG->getMachineNode(Opc, DL, MVT::Untyped, Op0, Op1Lo, Op1Hi);
     } else {
       SDValue Op1 = buildGPRPair(CurDAG, DL, MVT::Untyped, Op1Lo, Op1Hi);
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 7a2b9611683c6..a0e53d2e69c1f 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -21550,6 +21550,15 @@ static SDValue combineMinMaxToSat(SDNode *N,
   return SDValue();
 }
 
+// Returns true if the i32 pair (Lo, Hi) is the 64-bit sign-extension of the
+// i32 value Lo, i.e. Hi == (sra Lo, 31). Used to fold ADDD/SUBD of a
+// sign-extended operand into the WADDA/WSUBA widening accumulate nodes.
+static bool isI32SignExtended(SDValue Lo, SDValue Hi) {
+  return Hi.getOpcode() == ISD::SRA && Hi.getOperand(0) == Lo &&
+         isa<ConstantSDNode>(Hi.getOperand(1)) &&
+         Hi.getConstantOperandVal(1) == 31;
+}
+
 SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
                                                DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
@@ -21736,6 +21745,21 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
                       Op1Lo, Op1Hi, Op0Lo, DAG.getConstant(0, DL, MVT::i32));
       return DCI.CombineTo(N, Result.getValue(0), Result.getValue(1));
     }
+
+    // (ADDD lo, hi, x, sra(x, 31)) -> (WADDA lo, hi, x, 0)
+    if (isI32SignExtended(Op1Lo, Op1Hi)) {
+      SDValue Result =
+          DAG.getNode(RISCVISD::WADDA, DL, DAG.getVTList(MVT::i32, MVT::i32),
+                      Op0Lo, Op0Hi, Op1Lo, DAG.getConstant(0, DL, MVT::i32));
+      return DCI.CombineTo(N, Result.getValue(0), Result.getValue(1));
+    }
+    // (ADDD x, sra(x, 31), lo, hi) -> (WADDA lo, hi, x, 0)
+    if (isI32SignExtended(Op0Lo, Op0Hi)) {
+      SDValue Result =
+          DAG.getNode(RISCVISD::WADDA, DL, DAG.getVTList(MVT::i32, MVT::i32),
+                      Op1Lo, Op1Hi, Op0Lo, DAG.getConstant(0, DL, MVT::i32));
+      return DCI.CombineTo(N, Result.getValue(0), Result.getValue(1));
+    }
     break;
   }
   case RISCVISD::SUBD: {
@@ -21755,6 +21779,15 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
                       Op0Lo, Op0Hi, DAG.getConstant(0, DL, MVT::i32), Op1Lo);
       return DCI.CombineTo(N, Result.getValue(0), Result.getValue(1));
     }
+
+    // (SUBD lo, hi, x, sra(x, 31)) -> (WSUBA lo, hi, 0, x)
+    // WSUBA semantics: rd = rd + sext(rs1) - sext(rs2)
+    if (isI32SignExtended(Op1Lo, Op1Hi)) {
+      SDValue Result =
+          DAG.getNode(RISCVISD::WSUBA, DL, DAG.getVTList(MVT::i32, MVT::i32),
+                      Op0Lo, Op0Hi, DAG.getConstant(0, DL, MVT::i32), Op1Lo);
+      return DCI.CombineTo(N, Result.getValue(0), Result.getValue(1));
+    }
     break;
   }
   case RISCVISD::WADDAU: {
@@ -21836,6 +21869,57 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
     }
     break;
   }
+  case RISCVISD::WADDA: {
+    assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
+           "WADDA is only for RV32 with P extension");
+    SDValue Op0Lo = N->getOperand(0);
+    SDValue Op0Hi = N->getOperand(1);
+    SDValue Op1 = N->getOperand(2);
+    SDValue Op2 = N->getOperand(3);
+
+    // Fold a chained accumulate into the free second source slot.
+    if (isNullConstant(Op2) && Op0Lo.getNode() == Op0Hi.getNode() &&
+        Op0Lo.getResNo() == 0 && Op0Hi.getResNo() == 1 && Op0Lo.hasOneUse() &&
+        Op0Hi.hasOneUse()) {
+      // (WADDA (WADDA lo, hi, x, 0), y, 0) -> (WADDA lo, hi, x, y)
+      if (Op0Lo.getOpcode() == RISCVISD::WADDA &&
+          isNullConstant(Op0Lo.getOperand(3))) {
+        SDValue Result = DAG.getNode(
+            RISCVISD::WADDA, DL, DAG.getVTList(MVT::i32, MVT::i32),
+            Op0Lo.getOperand(0), Op0Lo.getOperand(1), Op0Lo.getOperand(2), Op1);
+        return DCI.CombineTo(N, Result.getValue(0), Result.getValue(1));
+      }
+      // (WADDA (WSUBA lo, hi, 0, a), b, 0) -> (WSUBA lo, hi, b, a)
+      if (Op0Lo.getOpcode() == RISCVISD::WSUBA &&
+          isNullConstant(Op0Lo.getOperand(2))) {
+        SDValue Result = DAG.getNode(
+            RISCVISD::WSUBA, DL, DAG.getVTList(MVT::i32, MVT::i32),
+            Op0Lo.getOperand(0), Op0Lo.getOperand(1), Op1, Op0Lo.getOperand(3));
+        return DCI.CombineTo(N, Result.getValue(0), Result.getValue(1));
+      }
+    }
+    break;
+  }
+  case RISCVISD::WSUBA: {
+    assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
+           "WSUBA is only for RV32 with P extension");
+    SDValue Op0Lo = N->getOperand(0);
+    SDValue Op0Hi = N->getOperand(1);
+    SDValue Op1 = N->getOperand(2);
+    SDValue Op2 = N->getOperand(3);
+
+    // (WSUBA (WADDA lo, hi, a, 0), 0, b) -> (WSUBA lo, hi, a, b)
+    if (isNullConstant(Op1) && Op0Lo.getOpcode() == RISCVISD::WADDA &&
+        Op0Lo.getNode() == Op0Hi.getNode() && Op0Lo.getResNo() == 0 &&
+        Op0Hi.getResNo() == 1 && Op0Lo.hasOneUse() && Op0Hi.hasOneUse() &&
+        isNullConstant(Op0Lo.getOperand(3))) {
+      SDValue Result = DAG.getNode(
+          RISCVISD::WSUBA, DL, DAG.getVTList(MVT::i32, MVT::i32),
+          Op0Lo.getOperand(0), Op0Lo.getOperand(1), Op0Lo.getOperand(2), Op2);
+      return DCI.CombineTo(N, Result.getValue(0), Result.getValue(1));
+    }
+    break;
+  }
   case RISCVISD::FMV_W_X_RV64: {
     // If the input to FMV_W_X_RV64 is just FMV_X_ANYEXTW_RV64 the the
     // conversion is unnecessary and can be replaced with the
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
index 60cdc9a3d2539..83095ae1985b4 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
@@ -1799,6 +1799,10 @@ def SDT_RISCVWideningAddSubAccumulate : SDTypeProfile<2, 4, [SDTCisVT<0, i32>,
 def riscv_waddau : RVSDNode<"WADDAU", SDT_RISCVWideningAddSubAccumulate>;
 // Widening sub accumulate unsigned: rd = rd + zext(rs1) - zext(rs2)
 def riscv_wsubau : RVSDNode<"WSUBAU", SDT_RISCVWideningAddSubAccumulate>;
+// Widening add accumulate signed: rd = rd + sext(rs1) + sext(rs2)
+def riscv_wadda : RVSDNode<"WADDA", SDT_RISCVWideningAddSubAccumulate>;
+// Widening sub accumulate signed: rd = rd + sext(rs1) - sext(rs2)
+def riscv_wsuba : RVSDNode<"WSUBA", SDT_RISCVWideningAddSubAccumulate>;
 
 def riscv_waddu : RVSDNode<"WADDU", SDTIntBinHiLoOp, [SDNPCommutative]>;
 def riscv_wsubu : RVSDNode<"WSUBU", SDTIntBinHiLoOp>;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index e5c8a5df1b54c..cf4e0c081a1b8 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -62761,6 +62761,39 @@ static SDValue combineNestedGF2P8AFFINEQB(SDNode *N, const SDLoc &DL,
                      DAG.getTargetConstant(FoldedImm, DL, MVT::i8));
 }
 
+// Fold: GF2P8AFFINEQB(X ^ Splat(C), Y, Imm)
+//    => GF2P8AFFINEQB(X, Y, (u8)GF2P8AFFINEQB(C, Y, Imm))
+// Reassociating a XOR (by permuting it in the same manner as the input) allows
+// it to be applied for free using the immediate.
+static SDValue combineXorOnGF2P8AFFINEQBOperand(SDNode *N, const SDLoc &DL,
+                                                SelectionDAG &DAG, EVT VT) {
+  using namespace SDPatternMatch;
+
+  unsigned MatrixWidth = 64;
+
+  SDValue X, Y, SplatOp;
+  APInt Imm, SplatVal, ConstUndef;
+  SmallVector<APInt> MatEltBits;
+
+  if (sd_match(N, m_TernaryOp(X86ISD::GF2P8AFFINEQB,
+                              m_Xor(m_Value(X), m_Value(SplatOp)), m_Value(Y),
+                              m_ConstInt(Imm))) &&
+      X86::isConstantSplat(SplatOp, SplatVal, /*AllowPartialUndefs=*/false) &&
+      getTargetConstantBitsFromNode(Y, MatrixWidth, ConstUndef, MatEltBits,
+                                    /*AllowWholeUndefs=*/false)) {
+    // Immediate is shared, so all matrices need to permute it in the same way
+    if (!llvm::all_equal(MatEltBits))
+      return SDValue();
+
+    APInt NewImm = getGFNIByteAffine(SplatVal, MatEltBits[0], Imm);
+
+    return DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, X, Y,
+                       DAG.getTargetConstant(NewImm, DL, MVT::i8));
+  }
+
+  return SDValue();
+}
+
 static SDValue combineGF2P8AFFINEQB(SDNode *N, SelectionDAG &DAG) {
   EVT VT = N->getValueType(0);
   SDLoc dl(N);
@@ -62768,6 +62801,9 @@ static SDValue combineGF2P8AFFINEQB(SDNode *N, SelectionDAG &DAG) {
   if (SDValue R = combineAndOnGF2P8AFFINEQBOperand(N, dl, DAG, VT))
     return R;
 
+  if (SDValue R = combineXorOnGF2P8AFFINEQBOperand(N, dl, DAG, VT))
+    return R;
+
   if (SDValue R = combineNestedGF2P8AFFINEQB(N, dl, DAG, VT))
     return R;
 
diff --git a/llvm/lib/Target/X86/X86InstrMisc.td b/llvm/lib/Target/X86/X86InstrMisc.td
index 84d034287de7b..14a2f477450a3 100644
--- a/llvm/lib/Target/X86/X86InstrMisc.td
+++ b/llvm/lib/Target/X86/X86InstrMisc.td
@@ -1295,6 +1295,20 @@ let Predicates = [HasBMI] in
                                        GR8:$src, sub_8bit)),
               sub_8bit)>;
 
+// BLSMSK only has GR32/GR64 forms. For an i8 operand we use INSERT_SUBREG
+// with IMPLICIT_DEF (any-extend) rather than SUBREG_TO_REG (zero-extend).
+// Although BLSMSK computes (src XOR (src-1)), dirty upper bits [31:8] cannot
+// corrupt bits [7:0] of the result: borrow from (src-1) only propagates
+// downward through contiguous zeros, and even when [7:0] is zero the
+// EXTRACT_SUBREG still yields the correct all-ones result. This matches the
+// same approach used for BLSI.
+let Predicates = [HasBMI] in
+  def : Pat<(xor GR8:$src, (add GR8:$src, -1)),
+            (EXTRACT_SUBREG
+              (BLSMSK32rr (INSERT_SUBREG (i32 (IMPLICIT_DEF)),
+                                         GR8:$src, sub_8bit)),
+              sub_8bit)>;
+
 multiclass Bmi4VOp3<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node,
                     X86FoldableSchedWrite sched, string Suffix = ""> {
   let SchedRW = [sched], Form = MRMSrcReg4VOp3 in
diff --git a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
index 3be54a710b0ec..dc83d21e643e9 100644
--- a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
+++ b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
@@ -817,6 +817,10 @@ static Function *promoteArguments(Function *F, FunctionAnalysisManager &FAM,
   if (!F->hasLocalLinkage())
     return nullptr;
 
+  // Ensure function definition is available for interprocedural analysis.
+  if (!F->isDefinitionExact())
+    return nullptr;
+
   // Don't promote arguments for variadic functions. Adding, removing, or
   // changing non-pack parameters can change the classification of pack
   // parameters. Frontends encode that classification at the call site in the
diff --git a/llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp b/llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp
index af94a1df2ac34..d3b8fe496a5cf 100644
--- a/llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp
+++ b/llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp
@@ -482,6 +482,12 @@ void DeadArgumentEliminationPass::surveyFunction(const Function &F) {
     return;
   }
 
+  // Ensure function definition is available for interprocedural analysis.
+  if (!F.isDefinitionExact()) {
+    markFrozen(F);
+    return;
+  }
+
   unsigned RetCount = numRetVals(&F);
 
   // Assume all return values are dead
diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
index 75d0f27169822..0618073387d97 100644
--- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp
+++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
@@ -1983,6 +1983,10 @@ OptimizeFunctions(Module &M,
     if (!F.hasLocalLinkage())
       continue;
 
+    // Ensure function definition is available for interprocedural analysis.
+    if (!F.isDefinitionExact())
+      continue;
+
     // If we have an inalloca parameter that we can safely remove the
     // inalloca attribute from, do so. This unlocks optimizations that
     // wouldn't be safe in the presence of inalloca.
diff --git a/llvm/lib/Transforms/IPO/MergeFunctions.cpp b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
index ac8a21ea24320..5c8624ba56a27 100644
--- a/llvm/lib/Transforms/IPO/MergeFunctions.cpp
+++ b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
@@ -428,6 +428,7 @@ static bool hasDistinctMetadataIntrinsic(const Function &F) {
 /// Check whether \p F is eligible for function merging.
 static bool isEligibleForMerging(Function &F) {
   return !F.isDeclaration() && !F.hasAvailableExternallyLinkage() &&
+         !F.hasFnAttribute(Attribute::NoIPA) &&
          !hasDistinctMetadataIntrinsic(F);
 }
 
@@ -473,7 +474,8 @@ template <typename FuncContainer> bool MergeFunctions::run(FuncContainer &M) {
       if (!I)
         continue;
       Function *F = cast<Function>(I);
-      if (!F->isDeclaration() && !F->hasAvailableExternallyLinkage()) {
+      if (!F->isDeclaration() && !F->hasAvailableExternallyLinkage() &&
+          !F->hasFnAttribute(Attribute::NoIPA)) {
         Changed |= insert(F);
       }
     }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index af10ac712886a..d5ae2d3f2e0ad 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -273,16 +273,6 @@ static unsigned getSelectFoldableOperands(BinaryOperator *I) {
 /// We have (select c, TI, FI), and we know that TI and FI have the same opcode.
 Instruction *InstCombinerImpl::foldSelectOpOp(SelectInst &SI, Instruction *TI,
                                               Instruction *FI) {
-  // Don't break up min/max patterns. The hasOneUse checks below prevent that
-  // for most cases, but vector min/max with bitcasts can be transformed. If the
-  // one-use restrictions are eased for other patterns, we still don't want to
-  // obfuscate min/max.
-  if ((match(&SI, m_SMin(m_Value(), m_Value())) ||
-       match(&SI, m_SMax(m_Value(), m_Value())) ||
-       match(&SI, m_UMin(m_Value(), m_Value())) ||
-       match(&SI, m_UMax(m_Value(), m_Value()))))
-    return nullptr;
-
   // If this is a cast from the same type, merge.
   Value *Cond = SI.getCondition();
   Type *CondTy = Cond->getType();
diff --git a/llvm/lib/Transforms/Scalar/NaryReassociate.cpp b/llvm/lib/Transforms/Scalar/NaryReassociate.cpp
index cf0d77dc11116..d24440646f766 100644
--- a/llvm/lib/Transforms/Scalar/NaryReassociate.cpp
+++ b/llvm/lib/Transforms/Scalar/NaryReassociate.cpp
@@ -270,28 +270,6 @@ bool NaryReassociatePass::doOneIteration(Function &F) {
   return Changed;
 }
 
-template <typename PredT>
-Instruction *
-NaryReassociatePass::matchAndReassociateMinOrMax(Instruction *I,
-                                                 SCEVUse &OrigSCEV) {
-  Value *LHS = nullptr;
-  Value *RHS = nullptr;
-
-  auto MinMaxMatcher =
-      MaxMin_match<ICmpInst, match_bind<Value>, match_bind<Value>, PredT>(
-          m_Value(LHS), m_Value(RHS));
-  if (match(I, MinMaxMatcher)) {
-    OrigSCEV = SE->getSCEV(I);
-    if (auto *NewMinMax = dyn_cast_or_null<Instruction>(
-            tryReassociateMinOrMax(I, MinMaxMatcher, LHS, RHS)))
-      return NewMinMax;
-    if (auto *NewMinMax = dyn_cast_or_null<Instruction>(
-            tryReassociateMinOrMax(I, MinMaxMatcher, RHS, LHS)))
-      return NewMinMax;
-  }
-  return nullptr;
-}
-
 Instruction *NaryReassociatePass::tryReassociate(Instruction *I,
                                                  SCEVUse &OrigSCEV) {
 
@@ -311,16 +289,11 @@ Instruction *NaryReassociatePass::tryReassociate(Instruction *I,
   }
 
   // Try to match signed/unsigned Min/Max.
-  Instruction *ResI = nullptr;
-  // TODO: Currently min/max reassociation is restricted to integer types only
-  // due to use of SCEVExpander which my introduce incompatible forms of min/max
-  // for pointer types.
-  if (I->getType()->isIntegerTy())
-    if ((ResI = matchAndReassociateMinOrMax<umin_pred_ty>(I, OrigSCEV)) ||
-        (ResI = matchAndReassociateMinOrMax<smin_pred_ty>(I, OrigSCEV)) ||
-        (ResI = matchAndReassociateMinOrMax<umax_pred_ty>(I, OrigSCEV)) ||
-        (ResI = matchAndReassociateMinOrMax<smax_pred_ty>(I, OrigSCEV)))
-      return ResI;
+  if (match(I, m_MaxOrMin(m_Value(), m_Value()))) {
+    OrigSCEV = SE->getSCEV(I);
+    return dyn_cast_or_null<Instruction>(
+        tryReassociateMinOrMax(cast<IntrinsicInst>(I)));
+  }
 
   return nullptr;
 }
@@ -442,7 +415,7 @@ NaryReassociatePass::tryReassociateGEPAtIndex(GetElementPtrInst *GEP,
   // sizeof(S) = 100 is indivisible by sizeof(int64) = 8.
   //
   // TODO: bail out on this case for now. We could emit uglygep.
-  if (IndexedSize % ElementSize != 0)
+  if (ElementSize == 0 || IndexedSize % ElementSize != 0)
     return nullptr;
 
   // NewGEP = &Candidate[RHS * (sizeof(IndexedType) / sizeof(Candidate[0])));
@@ -583,35 +556,34 @@ NaryReassociatePass::findClosestMatchingDominator(SCEVUse CandidateExpr,
   return nullptr;
 }
 
-template <typename MaxMinT> static SCEVTypes convertToSCEVType(MaxMinT &MM) {
-  if (std::is_same_v<smax_pred_ty, typename MaxMinT::PredType>)
+static SCEVTypes convertToSCEVType(Intrinsic::ID IntrinID) {
+  switch (IntrinID) {
+  case Intrinsic::smax:
     return scSMaxExpr;
-  else if (std::is_same_v<umax_pred_ty, typename MaxMinT::PredType>)
+  case Intrinsic::umax:
     return scUMaxExpr;
-  else if (std::is_same_v<smin_pred_ty, typename MaxMinT::PredType>)
+  case Intrinsic::smin:
     return scSMinExpr;
-  else if (std::is_same_v<umin_pred_ty, typename MaxMinT::PredType>)
+  case Intrinsic::umin:
     return scUMinExpr;
-
-  llvm_unreachable("Can't convert MinMax pattern to SCEV type");
-  return scUnknown;
+  default:
+    llvm_unreachable("Can't convert MinMax pattern to SCEV type");
+    return scUnknown;
+  }
 }
 
-// Parameters:
-//  I - instruction matched by MaxMinMatch matcher
-//  MaxMinMatch - min/max idiom matcher
-//  LHS - first operand of I
-//  RHS - second operand of I
-template <typename MaxMinT>
-Value *NaryReassociatePass::tryReassociateMinOrMax(Instruction *I,
-                                                   MaxMinT MaxMinMatch,
-                                                   Value *LHS, Value *RHS) {
-  Value *A = nullptr, *B = nullptr;
-  MaxMinT m_MaxMin(m_Value(A), m_Value(B));
-
-  if (!match(LHS, m_MaxMin))
+Value *NaryReassociatePass::tryReassociateMinOrMax(IntrinsicInst *I) {
+  Value *LHS = I->getArgOperand(0);
+  Value *RHS = I->getArgOperand(1);
+  if (auto *RHSI = dyn_cast<IntrinsicInst>(RHS);
+      RHSI && RHSI->getIntrinsicID() == I->getIntrinsicID())
+    std::swap(LHS, RHS);
+  auto *LHSI = dyn_cast<IntrinsicInst>(LHS);
+  if (!LHSI || LHSI->getIntrinsicID() != I->getIntrinsicID())
     return nullptr;
 
+  Value *A = LHSI->getArgOperand(0), *B = LHSI->getArgOperand(1);
+
   if (LHS->hasNUsesOrMore(3) ||
       // The optimization is profitable only if LHS can be removed in the end.
       // In other words LHS should be used (directly or indirectly) by I only.
@@ -623,7 +595,7 @@ Value *NaryReassociatePass::tryReassociateMinOrMax(Instruction *I,
   auto tryCombination = [&](Value *A, SCEVUse AExpr, Value *B, SCEVUse BExpr,
                             Value *C, SCEVUse CExpr) -> Value * {
     SmallVector<SCEVUse, 2> Ops1{BExpr, AExpr};
-    const SCEVTypes SCEVType = convertToSCEVType(m_MaxMin);
+    SCEVTypes SCEVType = convertToSCEVType(I->getIntrinsicID());
     SCEVUse R1Expr = SE->getMinMaxExpr(SCEVType, Ops1);
 
     Instruction *R1MinMax = findClosestMatchingDominator(R1Expr, I);
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 6e061d6925eb7..90be4c994c539 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -5306,9 +5306,10 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
 
 /// Try to canonicalize a homogeneous struct partition to a vector type.
 ///
-/// We can do this if all the elements of the struct are the same and tightly
-/// packed. This can sometimes eliminate allocas because structs cannot get
-/// promoted to LLVM values, but vectors can.
+/// We can do this if all the elements of the struct are the same and the
+/// corresponding vector has the same byte-level layout. This can sometimes
+/// eliminate allocas because structs cannot get promoted to LLVM values, but
+/// vectors can.
 ///
 /// We only apply this transformation when all users of the partition are memory
 /// intrinsics. Otherwise, if there is a load or store of some other type to the
@@ -5341,9 +5342,18 @@ static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
       !IsIntegralPointerTy)
     return nullptr;
 
+  // Ensure the struct is tightly packed so that the bit-layout is the same as
+  // the corresponding vector. For example, this prevents a miscompile for
+  // { i5, i5 }, which has padding after each i5 field, whereas <i5, i5> has
+  // tightly packed elements and trailing padding.
+  if (DL.getTypeSizeInBits(EltTy) != DL.getTypeAllocSizeInBits(EltTy))
+    return nullptr;
+
   auto *VTy = FixedVectorType::get(EltTy, NumElts);
   TypeSize StructSize = DL.getStructLayout(STy)->getSizeInBytes();
-  TypeSize VectorSize = DL.getTypeAllocSize(VTy);
+  TypeSize VectorSize = DL.getTypeStoreSize(VTy);
+  // After ruling out per-element padding, make sure a vector load/store
+  // covers the same number of bytes as the struct layout.
   if (StructSize != VectorSize)
     return nullptr;
 
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index 1ec5771bb819f..734596fb1154a 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -982,6 +982,7 @@ Function *CodeExtractor::constructFunctionDeclaration(
       case Attribute::NoFree:
       case Attribute::NoImplicitFloat:
       case Attribute::NoInline:
+      case Attribute::NoIPA:
       case Attribute::NoOutline:
       case Attribute::NonLazyBind:
       case Attribute::NoRedZone:
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d3e491247f93f..4c787179712f0 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28904,10 +28904,7 @@ class HorizontalReduction {
       return RecurKind::FMaximum;
     if (match(I, m_FMinimum(m_Value(), m_Value())))
       return RecurKind::FMinimum;
-    // This matches either cmp+select or intrinsics. SLP is expected to handle
-    // either form.
-    // TODO: If we are canonicalizing to intrinsics, we can remove several
-    //       special-case paths that deal with selects.
+    // TODO: remove several special-case paths that deal with selects.
     if (match(I, m_SMax(m_Value(), m_Value())))
       return RecurKind::SMax;
     if (match(I, m_SMin(m_Value(), m_Value())))
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 1f133cbaa95bb..0b55098e90c0e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -27,26 +27,21 @@ template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) {
   return P.match(V);
 }
 
-/// A match functor that can be used as a UnaryPredicate in functional
-/// algorithms like all_of.
-template <typename Val, typename Pattern> auto match_fn(const Pattern &P) {
-  return bind_back<match<Val, Pattern>>(P);
-}
-
 template <typename Pattern> bool match(VPUser *U, const Pattern &P) {
   auto *R = dyn_cast<VPRecipeBase>(U);
   return R && match(R, P);
 }
 
-/// Match functor for VPUser.
-template <typename Pattern> auto match_fn(const Pattern &P) {
-  return bind_back<match<Pattern>>(P);
-}
-
 template <typename Pattern> bool match(VPSingleDefRecipe *R, const Pattern &P) {
   return P.match(static_cast<const VPRecipeBase *>(R));
 }
 
+/// A match functor that can be used as a UnaryPredicate in functional
+/// algorithms like all_of.
+template <typename Pattern> auto match_fn(const Pattern &P) {
+  return [&P](auto *V) { return match(V, P); };
+}
+
 /// Match an arbitrary VPValue and ignore it.
 inline auto m_VPValue() { return m_Isa<VPValue>(); }
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 7e0f536a63a83..327fdf2332312 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -155,8 +155,7 @@ GEPNoWrapFlags vputils::getGEPFlagsForPtr(VPValue *Ptr) {
   while (auto *PtrVPI = dyn_cast<VPInstruction>(Ptr)) {
     unsigned Opcode = PtrVPI->getOpcode();
     if (Opcode == Instruction::GetElementPtr) {
-      if (any_of(drop_begin(PtrVPI->operands()),
-                 [](VPValue *Op) { return !match(Op, m_ZeroInt()); }))
+      if (!all_of(drop_begin(PtrVPI->operands()), match_fn(m_ZeroInt())))
         return PtrVPI->getGEPNoWrapFlags();
       Ptr = PtrVPI->getOperand(0);
       continue;
diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll
index 21712fae7eecd..f696f2dd12323 100644
--- a/llvm/test/Bitcode/attributes.ll
+++ b/llvm/test/Bitcode/attributes.ll
@@ -592,6 +592,11 @@ define void @dead_on_return_sized(ptr dead_on_return(4) %p) {
   ret void
 }
 
+; CHECK: define void @noipa() [[NOIPA:#[0-9]+]]
+define void @noipa() noipa {
+  ret void
+}
+
 ; CHECK: attributes #0 = { noreturn }
 ; CHECK: attributes #1 = { nounwind }
 ; CHECK: attributes #2 = { memory(none) }
@@ -654,4 +659,5 @@ define void @dead_on_return_sized(ptr dead_on_return(4) %p) {
 ; CHECK: attributes [[SKIPPROFILE]] = { skipprofile }
 ; CHECK: attributes [[OPTDEBUG]] = { optdebug }
 ; CHECK: attributes [[NODIVERGENCESOURCE]] = { nodivergencesource }
+; CHECK: attributes [[NOIPA]] = { noipa }
 ; CHECK: attributes #[[NOBUILTIN]] = { nobuiltin }
diff --git a/llvm/test/CodeGen/AArch64/aarch64-fmuladd.ll b/llvm/test/CodeGen/AArch64/aarch64-fmuladd.ll
new file mode 100644
index 0000000000000..0d906c351ad60
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-fmuladd.ll
@@ -0,0 +1,47 @@
+; RUN: llc -mtriple=aarch64                      -global-isel=0 %s -o - | FileCheck %s --check-prefix=CHECK,FP
+; RUN: llc -mtriple=aarch64                  -O0 -global-isel=1 %s -o - | FileCheck %s --check-prefix=CHECK,FP
+; RUN: llc -mtriple=aarch64 -mattr=-fp-armv8     -global-isel=0 %s -o - | FileCheck %s --check-prefix=CHECK,NOFP
+; RUN: llc -mtriple=aarch64 -mattr=-fp-armv8 -O0 -global-isel=1 %s -o - | FileCheck %s --check-prefix=CHECK,NOFP
+
+; Test that @llvm.fmuladd, with the semantics "multiply and add can be
+; fused or separate at the compiler's option", fuses them in the
+; normal case of hardware FP, but doesn't fuse them in AArch64 softfp,
+; on the theory that the libc fma function is likely slower than the
+; combination.
+;
+; However, @llvm.fma must fuse them regardless, so that _should_ emit
+; a call to fma in softfp modes.
+
+define float @whichever_float(float %0, float %1, float %2) {
+  %result = call float @llvm.fmuladd.f32(float %0, float %1, float %2)
+  ret float %result
+; CHECK-LABEL: {{^}}whichever_float:
+; FP: fmadd s0, s0, s1, s2
+; NOFP: bl __mulsf3
+; NOFP: bl __addsf3
+}
+
+define double @whichever_double(double %0, double %1, double %2) {
+  %result = call double @llvm.fmuladd.f64(double %0, double %1, double %2)
+  ret double %result
+; CHECK-LABEL: {{^}}whichever_double:
+; FP: fmadd d0, d0, d1, d2
+; NOFP: bl __muldf3
+; NOFP: bl __adddf3
+}
+
+define float @force_fma_float(float %0, float %1, float %2) {
+  %result = call float @llvm.fma.f32(float %0, float %1, float %2)
+  ret float %result
+; CHECK-LABEL: {{^}}force_fma_float:
+; FP: fmadd s0, s0, s1, s2
+; NOFP: bl fmaf
+}
+
+define double @force_fma_double(double %0, double %1, double %2) {
+  %result = call double @llvm.fma.f64(double %0, double %1, double %2)
+  ret double %result
+; CHECK-LABEL: {{^}}force_fma_double:
+; FP: fmadd d0, d0, d1, d2
+; NOFP: bl fma
+}
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reassoc-duplicates.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reassoc-duplicates.ll
new file mode 100644
index 0000000000000..f569b606c6d3f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reassoc-duplicates.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S --passes=complex-deinterleaving -mtriple=aarch64-unknown-linux-gnu --mattr=+complxnum < %s | FileCheck %s
+
+define <2 x double> @repeated_imaginary_addend(i1 %c) {
+; CHECK-LABEL: define <2 x double> @repeated_imaginary_addend(
+; CHECK-SAME: i1 [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[ACC_R:%.*]] = phi <2 x double> [ zeroinitializer, %[[ENTRY]] ], [ [[ACC_R_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[ACC_I:%.*]] = phi <2 x double> [ zeroinitializer, %[[ENTRY]] ], [ [[ACC_I_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[R_ADDEND:%.*]] = shufflevector <4 x double> zeroinitializer, <4 x double> zeroinitializer, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[ACC_R_NEXT]] = fadd reassoc contract <2 x double> [[R_ADDEND]], [[ACC_R]]
+; CHECK-NEXT:    [[I_ADDEND:%.*]] = shufflevector <4 x double> zeroinitializer, <4 x double> zeroinitializer, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[ACC_I_TMP:%.*]] = fadd reassoc contract <2 x double> [[ACC_I]], [[I_ADDEND]]
+; CHECK-NEXT:    [[ACC_I_NEXT]] = fadd reassoc contract <2 x double> [[I_ADDEND]], [[ACC_I_TMP]]
+; CHECK-NEXT:    br i1 [[C]], label %[[EXIT:.*]], label %[[VECTOR_BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[REAL_USE:%.*]] = fadd <2 x double> [[ACC_R_NEXT]], zeroinitializer
+; CHECK-NEXT:    ret <2 x double> [[ACC_I_NEXT]]
+;
+entry:
+  br label %vector.body
+
+vector.body:
+  %acc.r = phi <2 x double> [ zeroinitializer, %entry ], [ %acc.r.next, %vector.body ]
+  %acc.i = phi <2 x double> [ zeroinitializer, %entry ], [ %acc.i.next, %vector.body ]
+  %r.addend = shufflevector <4 x double> zeroinitializer, <4 x double> zeroinitializer, <2 x i32> <i32 0, i32 2>
+  %acc.r.next = fadd reassoc contract <2 x double> %r.addend, %acc.r
+  %i.addend = shufflevector <4 x double> zeroinitializer, <4 x double> zeroinitializer, <2 x i32> <i32 1, i32 3>
+  %acc.i.tmp = fadd reassoc contract <2 x double> %acc.i, %i.addend
+  %acc.i.next = fadd reassoc contract <2 x double> %i.addend, %acc.i.tmp
+  br i1 %c, label %exit, label %vector.body
+
+exit:
+  %real.use = fadd <2 x double> %acc.r.next, zeroinitializer
+  ret <2 x double> %acc.i.next
+}
diff --git a/llvm/test/CodeGen/AArch64/sve2-fixed-length-sra-flag.ll b/llvm/test/CodeGen/AArch64/sve2-fixed-length-sra-flag.ll
new file mode 100644
index 0000000000000..f813b84c726d1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2-fixed-length-sra-flag.ll
@@ -0,0 +1,28 @@
+; REQUIRES: asserts
+; RUN: llc -debug-only=isel < %s 2>&1 | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @usra_disjoint_or32xi8(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or32xi8:
+; CHECK:    nxv16i8 = USRA_ZZI_B disjoint {{.*}}, {{.*}}, TargetConstant:i32<4>
+  %va = load <32 x i8>, ptr %a
+  %vb = load <32 x i8>, ptr %b
+  %shift = lshr <32 x i8> %vb, splat(i8 4)
+  %res = or disjoint <32 x i8> %va, %shift
+  store <32 x i8> %res, ptr %a
+  ret void
+}
+
+define void @ssra_disjoint_or32xi8(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or32xi8:
+; CHECK:    nxv16i8 = SSRA_ZZI_B disjoint {{.*}}, {{.*}}, TargetConstant:i32<4>
+  %va = load <32 x i8>, ptr %a
+  %vb = load <32 x i8>, ptr %b
+  %shift = ashr <32 x i8> %vb, splat(i8 4)
+  %res = or disjoint <32 x i8> %va, %shift
+  store <32 x i8> %res, ptr %a
+  ret void
+}
+
+attributes #0 = { "target-features"="+sve,+sve2" }
diff --git a/llvm/test/CodeGen/AArch64/sve2-fixed-length-sra.ll b/llvm/test/CodeGen/AArch64/sve2-fixed-length-sra.ll
new file mode 100644
index 0000000000000..602de94cb47ba
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2-fixed-length-sra.ll
@@ -0,0 +1,832 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -aarch64-sve-vector-bits-min=256  < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256
+; RUN: llc -aarch64-sve-vector-bits-min=512  < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+
+target triple = "aarch64-unknown-linux-gnu"
+
+;
+; USRA disjoint OR
+;
+
+define <8 x i8> @usra_disjoint_or8xi8(<8 x i8> %a, <8 x i8> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or8xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    usra v0.8b, v1.8b, #4
+; CHECK-NEXT:    ret
+  %shift = lshr <8 x i8> %b, splat(i8 4)
+  %res = or disjoint <8 x i8> %a, %shift
+  ret <8 x i8> %res
+}
+
+define <16 x i8> @usra_disjoint_or16xi8(<16 x i8> %a, <16 x i8> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or16xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    usra v0.16b, v1.16b, #4
+; CHECK-NEXT:    ret
+  %shift = lshr <16 x i8> %b, splat(i8 4)
+  %res = or disjoint <16 x i8> %a, %shift
+  ret <16 x i8> %res
+}
+
+define void @usra_disjoint_or32xi8(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or32xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b, vl32
+; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT:    ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT:    usra z0.b, z1.b, #4
+; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <32 x i8>, ptr %a
+  %vb = load <32 x i8>, ptr %b
+  %shift = lshr <32 x i8> %vb, splat(i8 4)
+  %res = or disjoint <32 x i8> %va, %shift
+  store <32 x i8> %res, ptr %a
+  ret void
+}
+
+define void @usra_disjoint_or64xi8(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: usra_disjoint_or64xi8:
+; VBITS_GE_256:       // %bb.0:
+; VBITS_GE_256-NEXT:    ptrue p0.b, vl32
+; VBITS_GE_256-NEXT:    mov w8, #32 // =0x20
+; VBITS_GE_256-NEXT:    ld1b { z0.b }, p0/z, [x0, x8]
+; VBITS_GE_256-NEXT:    ld1b { z1.b }, p0/z, [x1, x8]
+; VBITS_GE_256-NEXT:    ld1b { z2.b }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    ld1b { z3.b }, p0/z, [x1]
+; VBITS_GE_256-NEXT:    usra z0.b, z1.b, #4
+; VBITS_GE_256-NEXT:    usra z2.b, z3.b, #4
+; VBITS_GE_256-NEXT:    st1b { z0.b }, p0, [x0, x8]
+; VBITS_GE_256-NEXT:    st1b { z2.b }, p0, [x0]
+; VBITS_GE_256-NEXT:    ret
+;
+; VBITS_GE_512-LABEL: usra_disjoint_or64xi8:
+; VBITS_GE_512:       // %bb.0:
+; VBITS_GE_512-NEXT:    ptrue p0.b, vl64
+; VBITS_GE_512-NEXT:    ld1b { z0.b }, p0/z, [x0]
+; VBITS_GE_512-NEXT:    ld1b { z1.b }, p0/z, [x1]
+; VBITS_GE_512-NEXT:    usra z0.b, z1.b, #4
+; VBITS_GE_512-NEXT:    st1b { z0.b }, p0, [x0]
+; VBITS_GE_512-NEXT:    ret
+  %va = load <64 x i8>, ptr %a
+  %vb = load <64 x i8>, ptr %b
+  %shift = lshr <64 x i8> %vb, splat(i8 4)
+  %res = or disjoint <64 x i8> %va, %shift
+  store <64 x i8> %res, ptr %a
+  ret void
+}
+
+define void @usra_disjoint_or128xi8(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: usra_disjoint_or128xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b, vl128
+; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT:    ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT:    usra z0.b, z1.b, #4
+; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <128 x i8>, ptr %a
+  %vb = load <128 x i8>, ptr %b
+  %shift = lshr <128 x i8> %vb, splat(i8 4)
+  %res = or disjoint <128 x i8> %va, %shift
+  store <128 x i8> %res, ptr %a
+  ret void
+}
+
+define void @usra_disjoint_or256xi8(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: usra_disjoint_or256xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b, vl256
+; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT:    ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT:    usra z0.b, z1.b, #4
+; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <256 x i8>, ptr %a
+  %vb = load <256 x i8>, ptr %b
+  %shift = lshr <256 x i8> %vb, splat(i8 4)
+  %res = or disjoint <256 x i8> %va, %shift
+  store <256 x i8> %res, ptr %a
+  ret void
+}
+
+define <4 x i16> @usra_disjoint_or4xi16(<4 x i16> %a, <4 x i16> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or4xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    usra v0.4h, v1.4h, #4
+; CHECK-NEXT:    ret
+  %shift = lshr <4 x i16> %b, splat(i16 4)
+  %res = or disjoint <4 x i16> %a, %shift
+  ret <4 x i16> %res
+}
+
+define <8 x i16> @usra_disjoint_or8xi16(<8 x i16> %a, <8 x i16> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or8xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    usra v0.8h, v1.8h, #4
+; CHECK-NEXT:    ret
+  %shift = lshr <8 x i16> %b, splat(i16 4)
+  %res = or disjoint <8 x i16> %a, %shift
+  ret <8 x i16> %res
+}
+
+define void @usra_disjoint_or16xi16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or16xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h, vl16
+; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT:    usra z0.h, z1.h, #4
+; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <16 x i16>, ptr %a
+  %vb = load <16 x i16>, ptr %b
+  %shift = lshr <16 x i16> %vb, splat(i16 4)
+  %res = or disjoint <16 x i16> %va, %shift
+  store <16 x i16> %res, ptr %a
+  ret void
+}
+
+define void @usra_disjoint_or32xi16(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: usra_disjoint_or32xi16:
+; VBITS_GE_256:       // %bb.0:
+; VBITS_GE_256-NEXT:    ptrue p0.h, vl16
+; VBITS_GE_256-NEXT:    mov x8, #16 // =0x10
+; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT:    ld1h { z1.h }, p0/z, [x1, x8, lsl #1]
+; VBITS_GE_256-NEXT:    ld1h { z2.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    ld1h { z3.h }, p0/z, [x1]
+; VBITS_GE_256-NEXT:    usra z0.h, z1.h, #4
+; VBITS_GE_256-NEXT:    usra z2.h, z3.h, #4
+; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT:    st1h { z2.h }, p0, [x0]
+; VBITS_GE_256-NEXT:    ret
+;
+; VBITS_GE_512-LABEL: usra_disjoint_or32xi16:
+; VBITS_GE_512:       // %bb.0:
+; VBITS_GE_512-NEXT:    ptrue p0.h, vl32
+; VBITS_GE_512-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_512-NEXT:    ld1h { z1.h }, p0/z, [x1]
+; VBITS_GE_512-NEXT:    usra z0.h, z1.h, #4
+; VBITS_GE_512-NEXT:    st1h { z0.h }, p0, [x0]
+; VBITS_GE_512-NEXT:    ret
+  %va = load <32 x i16>, ptr %a
+  %vb = load <32 x i16>, ptr %b
+  %shift = lshr <32 x i16> %vb, splat(i16 4)
+  %res = or disjoint <32 x i16> %va, %shift
+  store <32 x i16> %res, ptr %a
+  ret void
+}
+
+define void @usra_disjoint_or64xi16(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: usra_disjoint_or64xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h, vl64
+; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT:    usra z0.h, z1.h, #4
+; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <64 x i16>, ptr %a
+  %vb = load <64 x i16>, ptr %b
+  %shift = lshr <64 x i16> %vb, splat(i16 4)
+  %res = or disjoint <64 x i16> %va, %shift
+  store <64 x i16> %res, ptr %a
+  ret void
+}
+
+define void @usra_disjoint_or128xi16(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: usra_disjoint_or128xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h, vl128
+; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT:    usra z0.h, z1.h, #4
+; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <128 x i16>, ptr %a
+  %vb = load <128 x i16>, ptr %b
+  %shift = lshr <128 x i16> %vb, splat(i16 4)
+  %res = or disjoint <128 x i16> %va, %shift
+  store <128 x i16> %res, ptr %a
+  ret void
+}
+
+define <2 x i32> @usra_disjoint_or2xi32(<2 x i32> %a, <2 x i32> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or2xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    usra v0.2s, v1.2s, #4
+; CHECK-NEXT:    ret
+  %shift = lshr <2 x i32> %b, splat(i32 4)
+  %res = or disjoint <2 x i32> %a, %shift
+  ret <2 x i32> %res
+}
+
+define <4 x i32> @usra_disjoint_or4xi32(<4 x i32> %a, <4 x i32> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or4xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    usra v0.4s, v1.4s, #4
+; CHECK-NEXT:    ret
+  %shift = lshr <4 x i32> %b, splat(i32 4)
+  %res = or disjoint <4 x i32> %a, %shift
+  ret <4 x i32> %res
+}
+
+define void @usra_disjoint_or8xi32(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or8xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s, vl8
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT:    usra z0.s, z1.s, #4
+; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <8 x i32>, ptr %a
+  %vb = load <8 x i32>, ptr %b
+  %shift = lshr <8 x i32> %vb, splat(i32 4)
+  %res = or disjoint <8 x i32> %va, %shift
+  store <8 x i32> %res, ptr %a
+  ret void
+}
+
+define void @usra_disjoint_or16xi32(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: usra_disjoint_or16xi32:
+; VBITS_GE_256:       // %bb.0:
+; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
+; VBITS_GE_256-NEXT:    mov x8, #8 // =0x8
+; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    ld1w { z1.s }, p0/z, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT:    ld1w { z2.s }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    ld1w { z3.s }, p0/z, [x1]
+; VBITS_GE_256-NEXT:    usra z0.s, z1.s, #4
+; VBITS_GE_256-NEXT:    usra z2.s, z3.s, #4
+; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    st1w { z2.s }, p0, [x0]
+; VBITS_GE_256-NEXT:    ret
+;
+; VBITS_GE_512-LABEL: usra_disjoint_or16xi32:
+; VBITS_GE_512:       // %bb.0:
+; VBITS_GE_512-NEXT:    ptrue p0.s, vl16
+; VBITS_GE_512-NEXT:    ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512-NEXT:    ld1w { z1.s }, p0/z, [x1]
+; VBITS_GE_512-NEXT:    usra z0.s, z1.s, #4
+; VBITS_GE_512-NEXT:    st1w { z0.s }, p0, [x0]
+; VBITS_GE_512-NEXT:    ret
+  %va = load <16 x i32>, ptr %a
+  %vb = load <16 x i32>, ptr %b
+  %shift = lshr <16 x i32> %vb, splat(i32 4)
+  %res = or disjoint <16 x i32> %va, %shift
+  store <16 x i32> %res, ptr %a
+  ret void
+}
+
+define void @usra_disjoint_or32xi32(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: usra_disjoint_or32xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s, vl32
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT:    usra z0.s, z1.s, #4
+; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <32 x i32>, ptr %a
+  %vb = load <32 x i32>, ptr %b
+  %shift = lshr <32 x i32> %vb, splat(i32 4)
+  %res = or disjoint <32 x i32> %va, %shift
+  store <32 x i32> %res, ptr %a
+  ret void
+}
+
+define void @usra_disjoint_or64xi32(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: usra_disjoint_or64xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s, vl64
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT:    usra z0.s, z1.s, #4
+; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <64 x i32>, ptr %a
+  %vb = load <64 x i32>, ptr %b
+  %shift = lshr <64 x i32> %vb, splat(i32 4)
+  %res = or disjoint <64 x i32> %va, %shift
+  store <64 x i32> %res, ptr %a
+  ret void
+}
+
+define <1 x i64> @usra_disjoint_or1xi64(<1 x i64> %a, <1 x i64> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or1xi64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    usra d0, d1, #4
+; CHECK-NEXT:    ret
+  %shift = lshr <1 x i64> %b, splat(i64 4)
+  %res = or disjoint <1 x i64> %a, %shift
+  ret <1 x i64> %res
+}
+
+define <2 x i64> @usra_disjoint_or2xi64(<2 x i64> %a, <2 x i64> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or2xi64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    usra v0.2d, v1.2d, #4
+; CHECK-NEXT:    ret
+  %shift = lshr <2 x i64> %b, splat(i64 4)
+  %res = or disjoint <2 x i64> %a, %shift
+  ret <2 x i64> %res
+}
+
+define void @usra_disjoint_or4xi64(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: usra_disjoint_or4xi64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d, vl4
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT:    usra z0.d, z1.d, #4
+; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <4 x i64>, ptr %a
+  %vb = load <4 x i64>, ptr %b
+  %shift = lshr <4 x i64> %vb, splat(i64 4)
+  %res = or disjoint <4 x i64> %va, %shift
+  store <4 x i64> %res, ptr %a
+  ret void
+}
+
+define void @usra_disjoint_or8xi64(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: usra_disjoint_or8xi64:
+; VBITS_GE_256:       // %bb.0:
+; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
+; VBITS_GE_256-NEXT:    mov x8, #4 // =0x4
+; VBITS_GE_256-NEXT:    ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    ld1d { z1.d }, p0/z, [x1, x8, lsl #3]
+; VBITS_GE_256-NEXT:    ld1d { z2.d }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    ld1d { z3.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT:    usra z0.d, z1.d, #4
+; VBITS_GE_256-NEXT:    usra z2.d, z3.d, #4
+; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    st1d { z2.d }, p0, [x0]
+; VBITS_GE_256-NEXT:    ret
+;
+; VBITS_GE_512-LABEL: usra_disjoint_or8xi64:
+; VBITS_GE_512:       // %bb.0:
+; VBITS_GE_512-NEXT:    ptrue p0.d, vl8
+; VBITS_GE_512-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_512-NEXT:    ld1d { z1.d }, p0/z, [x1]
+; VBITS_GE_512-NEXT:    usra z0.d, z1.d, #4
+; VBITS_GE_512-NEXT:    st1d { z0.d }, p0, [x0]
+; VBITS_GE_512-NEXT:    ret
+  %va = load <8 x i64>, ptr %a
+  %vb = load <8 x i64>, ptr %b
+  %shift = lshr <8 x i64> %vb, splat(i64 4)
+  %res = or disjoint <8 x i64> %va, %shift
+  store <8 x i64> %res, ptr %a
+  ret void
+}
+
+define void @usra_disjoint_or16xi64(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: usra_disjoint_or16xi64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d, vl16
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT:    usra z0.d, z1.d, #4
+; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <16 x i64>, ptr %a
+  %vb = load <16 x i64>, ptr %b
+  %shift = lshr <16 x i64> %vb, splat(i64 4)
+  %res = or disjoint <16 x i64> %va, %shift
+  store <16 x i64> %res, ptr %a
+  ret void
+}
+
+define void @usra_disjoint_or32xi64(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: usra_disjoint_or32xi64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d, vl32
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT:    usra z0.d, z1.d, #4
+; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <32 x i64>, ptr %a
+  %vb = load <32 x i64>, ptr %b
+  %shift = lshr <32 x i64> %vb, splat(i64 4)
+  %res = or disjoint <32 x i64> %va, %shift
+  store <32 x i64> %res, ptr %a
+  ret void
+}
+
+;
+; SSRA disjoint OR
+;
+
+define <8 x i8> @ssra_disjoint_or8xi8(<8 x i8> %a, <8 x i8> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or8xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ssra v0.8b, v1.8b, #4
+; CHECK-NEXT:    ret
+  %shift = ashr <8 x i8> %b, splat(i8 4)
+  %res = or disjoint <8 x i8> %a, %shift
+  ret <8 x i8> %res
+}
+
+define <16 x i8> @ssra_disjoint_or16xi8(<16 x i8> %a, <16 x i8> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or16xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ssra v0.16b, v1.16b, #4
+; CHECK-NEXT:    ret
+  %shift = ashr <16 x i8> %b, splat(i8 4)
+  %res = or disjoint <16 x i8> %a, %shift
+  ret <16 x i8> %res
+}
+
+define void @ssra_disjoint_or32xi8(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or32xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b, vl32
+; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT:    ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT:    ssra z0.b, z1.b, #4
+; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <32 x i8>, ptr %a
+  %vb = load <32 x i8>, ptr %b
+  %shift = ashr <32 x i8> %vb, splat(i8 4)
+  %res = or disjoint <32 x i8> %va, %shift
+  store <32 x i8> %res, ptr %a
+  ret void
+}
+
+define void @ssra_disjoint_or64xi8(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: ssra_disjoint_or64xi8:
+; VBITS_GE_256:       // %bb.0:
+; VBITS_GE_256-NEXT:    ptrue p0.b, vl32
+; VBITS_GE_256-NEXT:    mov w8, #32 // =0x20
+; VBITS_GE_256-NEXT:    ld1b { z0.b }, p0/z, [x0, x8]
+; VBITS_GE_256-NEXT:    ld1b { z1.b }, p0/z, [x1, x8]
+; VBITS_GE_256-NEXT:    ld1b { z2.b }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    ld1b { z3.b }, p0/z, [x1]
+; VBITS_GE_256-NEXT:    ssra z0.b, z1.b, #4
+; VBITS_GE_256-NEXT:    ssra z2.b, z3.b, #4
+; VBITS_GE_256-NEXT:    st1b { z0.b }, p0, [x0, x8]
+; VBITS_GE_256-NEXT:    st1b { z2.b }, p0, [x0]
+; VBITS_GE_256-NEXT:    ret
+;
+; VBITS_GE_512-LABEL: ssra_disjoint_or64xi8:
+; VBITS_GE_512:       // %bb.0:
+; VBITS_GE_512-NEXT:    ptrue p0.b, vl64
+; VBITS_GE_512-NEXT:    ld1b { z0.b }, p0/z, [x0]
+; VBITS_GE_512-NEXT:    ld1b { z1.b }, p0/z, [x1]
+; VBITS_GE_512-NEXT:    ssra z0.b, z1.b, #4
+; VBITS_GE_512-NEXT:    st1b { z0.b }, p0, [x0]
+; VBITS_GE_512-NEXT:    ret
+  %va = load <64 x i8>, ptr %a
+  %vb = load <64 x i8>, ptr %b
+  %shift = ashr <64 x i8> %vb, splat(i8 4)
+  %res = or disjoint <64 x i8> %va, %shift
+  store <64 x i8> %res, ptr %a
+  ret void
+}
+
+define void @ssra_disjoint_or128xi8(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or128xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b, vl128
+; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT:    ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT:    ssra z0.b, z1.b, #4
+; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <128 x i8>, ptr %a
+  %vb = load <128 x i8>, ptr %b
+  %shift = ashr <128 x i8> %vb, splat(i8 4)
+  %res = or disjoint <128 x i8> %va, %shift
+  store <128 x i8> %res, ptr %a
+  ret void
+}
+
+define void @ssra_disjoint_or256xi8(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or256xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b, vl256
+; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT:    ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT:    ssra z0.b, z1.b, #4
+; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <256 x i8>, ptr %a
+  %vb = load <256 x i8>, ptr %b
+  %shift = ashr <256 x i8> %vb, splat(i8 4)
+  %res = or disjoint <256 x i8> %va, %shift
+  store <256 x i8> %res, ptr %a
+  ret void
+}
+
+define <4 x i16> @ssra_disjoint_or4xi16(<4 x i16> %a, <4 x i16> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or4xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ssra v0.4h, v1.4h, #4
+; CHECK-NEXT:    ret
+  %shift = ashr <4 x i16> %b, splat(i16 4)
+  %res = or disjoint <4 x i16> %a, %shift
+  ret <4 x i16> %res
+}
+
+define <8 x i16> @ssra_disjoint_or8xi16(<8 x i16> %a, <8 x i16> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or8xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ssra v0.8h, v1.8h, #4
+; CHECK-NEXT:    ret
+  %shift = ashr <8 x i16> %b, splat(i16 4)
+  %res = or disjoint <8 x i16> %a, %shift
+  ret <8 x i16> %res
+}
+
+define void @ssra_disjoint_or16xi16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or16xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h, vl16
+; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT:    ssra z0.h, z1.h, #4
+; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <16 x i16>, ptr %a
+  %vb = load <16 x i16>, ptr %b
+  %shift = ashr <16 x i16> %vb, splat(i16 4)
+  %res = or disjoint <16 x i16> %va, %shift
+  store <16 x i16> %res, ptr %a
+  ret void
+}
+
+define void @ssra_disjoint_or32xi16(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: ssra_disjoint_or32xi16:
+; VBITS_GE_256:       // %bb.0:
+; VBITS_GE_256-NEXT:    ptrue p0.h, vl16
+; VBITS_GE_256-NEXT:    mov x8, #16 // =0x10
+; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT:    ld1h { z1.h }, p0/z, [x1, x8, lsl #1]
+; VBITS_GE_256-NEXT:    ld1h { z2.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    ld1h { z3.h }, p0/z, [x1]
+; VBITS_GE_256-NEXT:    ssra z0.h, z1.h, #4
+; VBITS_GE_256-NEXT:    ssra z2.h, z3.h, #4
+; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT:    st1h { z2.h }, p0, [x0]
+; VBITS_GE_256-NEXT:    ret
+;
+; VBITS_GE_512-LABEL: ssra_disjoint_or32xi16:
+; VBITS_GE_512:       // %bb.0:
+; VBITS_GE_512-NEXT:    ptrue p0.h, vl32
+; VBITS_GE_512-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_512-NEXT:    ld1h { z1.h }, p0/z, [x1]
+; VBITS_GE_512-NEXT:    ssra z0.h, z1.h, #4
+; VBITS_GE_512-NEXT:    st1h { z0.h }, p0, [x0]
+; VBITS_GE_512-NEXT:    ret
+  %va = load <32 x i16>, ptr %a
+  %vb = load <32 x i16>, ptr %b
+  %shift = ashr <32 x i16> %vb, splat(i16 4)
+  %res = or disjoint <32 x i16> %va, %shift
+  store <32 x i16> %res, ptr %a
+  ret void
+}
+
+define void @ssra_disjoint_or64xi16(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or64xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h, vl64
+; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT:    ssra z0.h, z1.h, #4
+; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <64 x i16>, ptr %a
+  %vb = load <64 x i16>, ptr %b
+  %shift = ashr <64 x i16> %vb, splat(i16 4)
+  %res = or disjoint <64 x i16> %va, %shift
+  store <64 x i16> %res, ptr %a
+  ret void
+}
+
+define void @ssra_disjoint_or128xi16(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or128xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h, vl128
+; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT:    ssra z0.h, z1.h, #4
+; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <128 x i16>, ptr %a
+  %vb = load <128 x i16>, ptr %b
+  %shift = ashr <128 x i16> %vb, splat(i16 4)
+  %res = or disjoint <128 x i16> %va, %shift
+  store <128 x i16> %res, ptr %a
+  ret void
+}
+
+define <2 x i32> @ssra_disjoint_or2xi32(<2 x i32> %a, <2 x i32> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or2xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ssra v0.2s, v1.2s, #4
+; CHECK-NEXT:    ret
+  %shift = ashr <2 x i32> %b, splat(i32 4)
+  %res = or disjoint <2 x i32> %a, %shift
+  ret <2 x i32> %res
+}
+
+define <4 x i32> @ssra_disjoint_or4xi32(<4 x i32> %a, <4 x i32> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or4xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ssra v0.4s, v1.4s, #4
+; CHECK-NEXT:    ret
+  %shift = ashr <4 x i32> %b, splat(i32 4)
+  %res = or disjoint <4 x i32> %a, %shift
+  ret <4 x i32> %res
+}
+
+define void @ssra_disjoint_or8xi32(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or8xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s, vl8
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT:    ssra z0.s, z1.s, #4
+; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <8 x i32>, ptr %a
+  %vb = load <8 x i32>, ptr %b
+  %shift = ashr <8 x i32> %vb, splat(i32 4)
+  %res = or disjoint <8 x i32> %va, %shift
+  store <8 x i32> %res, ptr %a
+  ret void
+}
+
+define void @ssra_disjoint_or16xi32(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: ssra_disjoint_or16xi32:
+; VBITS_GE_256:       // %bb.0:
+; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
+; VBITS_GE_256-NEXT:    mov x8, #8 // =0x8
+; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    ld1w { z1.s }, p0/z, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT:    ld1w { z2.s }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    ld1w { z3.s }, p0/z, [x1]
+; VBITS_GE_256-NEXT:    ssra z0.s, z1.s, #4
+; VBITS_GE_256-NEXT:    ssra z2.s, z3.s, #4
+; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    st1w { z2.s }, p0, [x0]
+; VBITS_GE_256-NEXT:    ret
+;
+; VBITS_GE_512-LABEL: ssra_disjoint_or16xi32:
+; VBITS_GE_512:       // %bb.0:
+; VBITS_GE_512-NEXT:    ptrue p0.s, vl16
+; VBITS_GE_512-NEXT:    ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512-NEXT:    ld1w { z1.s }, p0/z, [x1]
+; VBITS_GE_512-NEXT:    ssra z0.s, z1.s, #4
+; VBITS_GE_512-NEXT:    st1w { z0.s }, p0, [x0]
+; VBITS_GE_512-NEXT:    ret
+  %va = load <16 x i32>, ptr %a
+  %vb = load <16 x i32>, ptr %b
+  %shift = ashr <16 x i32> %vb, splat(i32 4)
+  %res = or disjoint <16 x i32> %va, %shift
+  store <16 x i32> %res, ptr %a
+  ret void
+}
+
+define void @ssra_disjoint_or32xi32(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or32xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s, vl32
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT:    ssra z0.s, z1.s, #4
+; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <32 x i32>, ptr %a
+  %vb = load <32 x i32>, ptr %b
+  %shift = ashr <32 x i32> %vb, splat(i32 4)
+  %res = or disjoint <32 x i32> %va, %shift
+  store <32 x i32> %res, ptr %a
+  ret void
+}
+
+define void @ssra_disjoint_or64xi32(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or64xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s, vl64
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT:    ssra z0.s, z1.s, #4
+; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <64 x i32>, ptr %a
+  %vb = load <64 x i32>, ptr %b
+  %shift = ashr <64 x i32> %vb, splat(i32 4)
+  %res = or disjoint <64 x i32> %va, %shift
+  store <64 x i32> %res, ptr %a
+  ret void
+}
+
+define <1 x i64> @ssra_disjoint_or1xi64(<1 x i64> %a, <1 x i64> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or1xi64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ssra d0, d1, #4
+; CHECK-NEXT:    ret
+  %shift = ashr <1 x i64> %b, splat(i64 4)
+  %res = or disjoint <1 x i64> %a, %shift
+  ret <1 x i64> %res
+}
+
+define <2 x i64> @ssra_disjoint_or2xi64(<2 x i64> %a, <2 x i64> %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or2xi64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ssra v0.2d, v1.2d, #4
+; CHECK-NEXT:    ret
+  %shift = ashr <2 x i64> %b, splat(i64 4)
+  %res = or disjoint <2 x i64> %a, %shift
+  ret <2 x i64> %res
+}
+
+define void @ssra_disjoint_or4xi64(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or4xi64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d, vl4
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT:    ssra z0.d, z1.d, #4
+; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <4 x i64>, ptr %a
+  %vb = load <4 x i64>, ptr %b
+  %shift = ashr <4 x i64> %vb, splat(i64 4)
+  %res = or disjoint <4 x i64> %va, %shift
+  store <4 x i64> %res, ptr %a
+  ret void
+}
+
+define void @ssra_disjoint_or8xi64(ptr %a, ptr %b) #0 {
+; VBITS_GE_256-LABEL: ssra_disjoint_or8xi64:
+; VBITS_GE_256:       // %bb.0:
+; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
+; VBITS_GE_256-NEXT:    mov x8, #4 // =0x4
+; VBITS_GE_256-NEXT:    ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    ld1d { z1.d }, p0/z, [x1, x8, lsl #3]
+; VBITS_GE_256-NEXT:    ld1d { z2.d }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    ld1d { z3.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT:    ssra z0.d, z1.d, #4
+; VBITS_GE_256-NEXT:    ssra z2.d, z3.d, #4
+; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    st1d { z2.d }, p0, [x0]
+; VBITS_GE_256-NEXT:    ret
+;
+; VBITS_GE_512-LABEL: ssra_disjoint_or8xi64:
+; VBITS_GE_512:       // %bb.0:
+; VBITS_GE_512-NEXT:    ptrue p0.d, vl8
+; VBITS_GE_512-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_512-NEXT:    ld1d { z1.d }, p0/z, [x1]
+; VBITS_GE_512-NEXT:    ssra z0.d, z1.d, #4
+; VBITS_GE_512-NEXT:    st1d { z0.d }, p0, [x0]
+; VBITS_GE_512-NEXT:    ret
+  %va = load <8 x i64>, ptr %a
+  %vb = load <8 x i64>, ptr %b
+  %shift = ashr <8 x i64> %vb, splat(i64 4)
+  %res = or disjoint <8 x i64> %va, %shift
+  store <8 x i64> %res, ptr %a
+  ret void
+}
+
+define void @ssra_disjoint_or16xi64(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or16xi64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d, vl16
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT:    ssra z0.d, z1.d, #4
+; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <16 x i64>, ptr %a
+  %vb = load <16 x i64>, ptr %b
+  %shift = ashr <16 x i64> %vb, splat(i64 4)
+  %res = or disjoint <16 x i64> %va, %shift
+  store <16 x i64> %res, ptr %a
+  ret void
+}
+
+define void @ssra_disjoint_or32xi64(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: ssra_disjoint_or32xi64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d, vl32
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT:    ssra z0.d, z1.d, #4
+; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
+; CHECK-NEXT:    ret
+  %va = load <32 x i64>, ptr %a
+  %vb = load <32 x i64>, ptr %b
+  %shift = ashr <32 x i64> %vb, splat(i64 4)
+  %res = or disjoint <32 x i64> %va, %shift
+  store <32 x i64> %res, ptr %a
+  ret void
+}
+
+attributes #0 = { "target-features"="+sve,+sve2" }
diff --git a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
index 26c814ba4f621..3871d970ff578 100644
--- a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
@@ -2833,11 +2833,11 @@ define i32 @test_widening_instr_mull(ptr %p1, ptr %p2, i32 %h) {
 ; CHECK-BE-NEXT:  .LBB24_1: // %loop
 ; CHECK-BE-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-BE-NEXT:    ld1 { v0.16b }, [x1], #16
-; CHECK-BE-NEXT:    add x8, x0, #16
 ; CHECK-BE-NEXT:    ld1 { v1.8h }, [x0]
-; CHECK-BE-NEXT:    ld1 { v3.8h }, [x8]
+; CHECK-BE-NEXT:    add x8, x0, #16
 ; CHECK-BE-NEXT:    add x9, x0, #48
 ; CHECK-BE-NEXT:    add x10, x0, #32
+; CHECK-BE-NEXT:    ld1 { v3.8h }, [x8]
 ; CHECK-BE-NEXT:    subs w2, w2, #1
 ; CHECK-BE-NEXT:    ushll v2.8h, v0.8b, #0
 ; CHECK-BE-NEXT:    ushll2 v0.8h, v0.16b, #0
@@ -3098,11 +3098,11 @@ define i32 @test_widening_instr_mull_2(ptr %p1, ptr %p2, i32 %h) {
 ; CHECK-BE-NEXT:    ld1 { v17.4s }, [x8]
 ; CHECK-BE-NEXT:    ld1 { v18.4s }, [x9]
 ; CHECK-BE-NEXT:    ld1 { v19.4s }, [x10]
-; CHECK-BE-NEXT:    subs w2, w2, #1
 ; CHECK-BE-NEXT:    tbl v5.16b, { v4.16b }, v1.16b
 ; CHECK-BE-NEXT:    tbl v6.16b, { v4.16b }, v3.16b
 ; CHECK-BE-NEXT:    tbl v7.16b, { v4.16b }, v2.16b
 ; CHECK-BE-NEXT:    tbl v4.16b, { v4.16b }, v0.16b
+; CHECK-BE-NEXT:    subs w2, w2, #1
 ; CHECK-BE-NEXT:    rev32 v5.16b, v5.16b
 ; CHECK-BE-NEXT:    rev32 v6.16b, v6.16b
 ; CHECK-BE-NEXT:    rev32 v7.16b, v7.16b
@@ -3170,18 +3170,18 @@ define i32 @mul_zext_16i8_sext_16i8(ptr %p1, ptr %p2, i32 %h) {
 ; CHECK-BE:       // %bb.0: // %entry
 ; CHECK-BE-NEXT:  .LBB27_1: // %loop
 ; CHECK-BE-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-BE-NEXT:    ld1 { v0.16b }, [x1], #16
+; CHECK-BE-NEXT:    ld1 { v0.16b }, [x0]
+; CHECK-BE-NEXT:    ld1 { v1.16b }, [x1], #16
 ; CHECK-BE-NEXT:    add x8, x0, #48
-; CHECK-BE-NEXT:    ld1 { v1.16b }, [x0]
 ; CHECK-BE-NEXT:    subs w2, w2, #1
-; CHECK-BE-NEXT:    sshll2 v2.8h, v1.16b, #0
-; CHECK-BE-NEXT:    ushll2 v3.8h, v0.16b, #0
-; CHECK-BE-NEXT:    sshll v1.8h, v1.8b, #0
-; CHECK-BE-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-BE-NEXT:    sshll2 v2.8h, v0.16b, #0
+; CHECK-BE-NEXT:    ushll2 v3.8h, v1.16b, #0
+; CHECK-BE-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-BE-NEXT:    ushll v1.8h, v1.8b, #0
 ; CHECK-BE-NEXT:    smull2 v4.4s, v2.8h, v3.8h
 ; CHECK-BE-NEXT:    smull v2.4s, v2.4h, v3.4h
-; CHECK-BE-NEXT:    smull v3.4s, v1.4h, v0.4h
-; CHECK-BE-NEXT:    smull2 v0.4s, v1.8h, v0.8h
+; CHECK-BE-NEXT:    smull v3.4s, v0.4h, v1.4h
+; CHECK-BE-NEXT:    smull2 v0.4s, v0.8h, v1.8h
 ; CHECK-BE-NEXT:    st1 { v4.4s }, [x8]
 ; CHECK-BE-NEXT:    add x8, x0, #32
 ; CHECK-BE-NEXT:    st1 { v3.4s }, [x0]
@@ -3244,11 +3244,11 @@ define i32 @mul_zext_16i8_sext_16i16(ptr %p1, ptr %p2, i32 %h) {
 ; CHECK-BE-NEXT:  .LBB28_1: // %loop
 ; CHECK-BE-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-BE-NEXT:    ld1 { v0.16b }, [x1], #16
-; CHECK-BE-NEXT:    add x8, x0, #16
 ; CHECK-BE-NEXT:    ld1 { v1.8h }, [x0]
-; CHECK-BE-NEXT:    ld1 { v3.8h }, [x8]
+; CHECK-BE-NEXT:    add x8, x0, #16
 ; CHECK-BE-NEXT:    add x9, x0, #48
 ; CHECK-BE-NEXT:    add x10, x0, #32
+; CHECK-BE-NEXT:    ld1 { v3.8h }, [x8]
 ; CHECK-BE-NEXT:    subs w2, w2, #1
 ; CHECK-BE-NEXT:    ushll v2.8h, v0.8b, #0
 ; CHECK-BE-NEXT:    ushll2 v0.8h, v0.16b, #0
diff --git a/llvm/test/CodeGen/NVPTX/address-folder.ll b/llvm/test/CodeGen/NVPTX/address-folder.ll
new file mode 100644
index 0000000000000..813fd2c439651
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/address-folder.ll
@@ -0,0 +1,111 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s | FileCheck %s
+; RUN: llc -O0 < %s | FileCheck %s --check-prefix=O0
+
+target triple = "nvptx64-nvidia-cuda"
+
+; A byval kernel parameter loaded at a constant offset from a block other than
+; the one that defines its address must still be read directly from parameter
+; space (ld.param [param+off]), not via a materialized base register.
+define ptx_kernel void @test_kernel_cross_block(ptr byval([2 x i64]) align 8 %r, i64 %n, ptr addrspace(1) %out) {
+; CHECK-LABEL: test_kernel_cross_block(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd2, [test_kernel_cross_block_param_1];
+; CHECK-NEXT:    setp.lt.s64 %p1, %rd2, 1;
+; CHECK-NEXT:    @%p1 bra $L__BB0_2;
+; CHECK-NEXT:  // %bb.1: // %body
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_kernel_cross_block_param_2];
+; CHECK-NEXT:    ld.param.b64 %rd3, [test_kernel_cross_block_param_0];
+; CHECK-NEXT:    ld.param.b64 %rd4, [test_kernel_cross_block_param_0+8];
+; CHECK-NEXT:    mul.lo.s64 %rd5, %rd3, %rd4;
+; CHECK-NEXT:    st.global.b64 [%rd1], %rd5;
+; CHECK-NEXT:  $L__BB0_2: // %done
+; CHECK-NEXT:    ret;
+;
+; O0-LABEL: test_kernel_cross_block(
+; O0:       {
+; O0-NEXT:    .reg .pred %p<2>;
+; O0-NEXT:    .reg .b64 %rd<7>;
+; O0-EMPTY:
+; O0-NEXT:  // %bb.0:
+; O0-NEXT:    ld.param.b64 %rd3, [test_kernel_cross_block_param_2];
+; O0-NEXT:    ld.param.b64 %rd2, [test_kernel_cross_block_param_1];
+; O0-NEXT:    mov.b64 %rd1, test_kernel_cross_block_param_0;
+; O0-NEXT:    setp.lt.s64 %p1, %rd2, 1;
+; O0-NEXT:    @%p1 bra $L__BB0_2;
+; O0-NEXT:    bra.uni $L__BB0_1;
+; O0-NEXT:  $L__BB0_1: // %body
+; O0-NEXT:    ld.param.b64 %rd4, [%rd1];
+; O0-NEXT:    ld.param.b64 %rd5, [%rd1+8];
+; O0-NEXT:    mul.lo.s64 %rd6, %rd4, %rd5;
+; O0-NEXT:    st.global.b64 [%rd3], %rd6;
+; O0-NEXT:    bra.uni $L__BB0_2;
+; O0-NEXT:  $L__BB0_2: // %done
+; O0-NEXT:    ret;
+  %c = icmp sgt i64 %n, 0
+  br i1 %c, label %body, label %done
+body:
+  %v0 = load i64, ptr %r, align 8
+  %p1 = getelementptr inbounds i8, ptr %r, i64 8
+  %v1 = load i64, ptr %p1, align 8
+  %m = mul i64 %v0, %v1
+  store i64 %m, ptr addrspace(1) %out, align 8
+  br label %done
+done:
+  ret void
+}
+
+; When the address is also used for arithmetic (here a variable offset) it is
+; not just loaded, so the `mov` must be kept rather than folded into the load.
+define ptx_kernel void @test_kernel_var_offset(ptr byval([8 x i64]) align 8 %r, i64 %n, ptr addrspace(1) %out) {
+; CHECK-LABEL: test_kernel_var_offset(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd4, [test_kernel_var_offset_param_1];
+; CHECK-NEXT:    setp.lt.s64 %p1, %rd4, 1;
+; CHECK-NEXT:    @%p1 bra $L__BB1_2;
+; CHECK-NEXT:  // %bb.1: // %body
+; CHECK-NEXT:    ld.param.b64 %rd2, [test_kernel_var_offset_param_2];
+; CHECK-NEXT:    mov.b64 %rd3, test_kernel_var_offset_param_0;
+; CHECK-NEXT:    add.s64 %rd1, %rd3, %rd4;
+; CHECK-NEXT:    ld.param.b64 %rd5, [%rd1];
+; CHECK-NEXT:    st.global.b64 [%rd2], %rd5;
+; CHECK-NEXT:  $L__BB1_2: // %done
+; CHECK-NEXT:    ret;
+;
+; O0-LABEL: test_kernel_var_offset(
+; O0:       {
+; O0-NEXT:    .reg .pred %p<2>;
+; O0-NEXT:    .reg .b64 %rd<6>;
+; O0-EMPTY:
+; O0-NEXT:  // %bb.0:
+; O0-NEXT:    ld.param.b64 %rd3, [test_kernel_var_offset_param_2];
+; O0-NEXT:    ld.param.b64 %rd2, [test_kernel_var_offset_param_1];
+; O0-NEXT:    mov.b64 %rd1, test_kernel_var_offset_param_0;
+; O0-NEXT:    setp.lt.s64 %p1, %rd2, 1;
+; O0-NEXT:    @%p1 bra $L__BB1_2;
+; O0-NEXT:    bra.uni $L__BB1_1;
+; O0-NEXT:  $L__BB1_1: // %body
+; O0-NEXT:    add.s64 %rd4, %rd1, %rd2;
+; O0-NEXT:    ld.param.b64 %rd5, [%rd4];
+; O0-NEXT:    st.global.b64 [%rd3], %rd5;
+; O0-NEXT:    bra.uni $L__BB1_2;
+; O0-NEXT:  $L__BB1_2: // %done
+; O0-NEXT:    ret;
+  %c = icmp sgt i64 %n, 0
+  br i1 %c, label %body, label %done
+body:
+  %p = getelementptr inbounds i8, ptr %r, i64 %n
+  %v = load i64, ptr %p, align 8
+  store i64 %v, ptr addrspace(1) %out, align 8
+  br label %done
+done:
+  ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/address-folder.mir b/llvm/test/CodeGen/NVPTX/address-folder.mir
new file mode 100644
index 0000000000000..ac50925579506
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/address-folder.mir
@@ -0,0 +1,60 @@
+# RUN: llc -mtriple=nvptx64-nvidia-cuda -run-pass=nvptx-address-folder -verify-machineinstrs -o - %s | FileCheck %s
+
+--- |
+  target triple = "nvptx64-nvidia-cuda"
+
+  @g = addrspace(1) global [4 x i64] zeroinitializer
+  @g_shared = addrspace(3) global [4 x i64] zeroinitializer
+
+  define void @fold_global() { ret void }
+  define void @skip_shared() { ret void }
+  define void @fold_partial() { ret void }
+...
+---
+# A global address materialized into a register is folded into the address
+# operand of loads and stores, and the dead `mov` is removed.
+# CHECK-LABEL: name: fold_global
+# CHECK-NOT: MOV_B64_sym
+# CHECK: %1:b64 = LD_i64 0, 0, 1, 3, 64, -1, @g, 0
+# CHECK: %2:b64 = LD_i64 0, 0, 1, 3, 64, -1, @g, 8
+# CHECK: ST_i64 %1, 0, 0, 1, 64, @g, 16
+name: fold_global
+tracksRegLiveness: true
+body: |
+  bb.0:
+    %0:b64 = MOV_B64_sym @g
+    %1:b64 = LD_i64 0, 0, 1, 3, 64, -1, %0, 0 :: (load (s64), addrspace 1)
+    %2:b64 = LD_i64 0, 0, 1, 3, 64, -1, %0, 8 :: (load (s64), addrspace 1)
+    ST_i64 %1, 0, 0, 1, 64, %0, 16 :: (store (s64), addrspace 1)
+    Return
+...
+---
+# Shared-memory accesses are not folded: the `mov` of the shared symbol must
+# stay CSE-able rather than be duplicated into every access.
+# CHECK-LABEL: name: skip_shared
+# CHECK: %0:b64 = MOV_B64_sym @g_shared
+# CHECK: %1:b64 = LD_i64 0, 0, 3, 3, 64, -1, %0, 0
+name: skip_shared
+tracksRegLiveness: true
+body: |
+  bb.0:
+    %0:b64 = MOV_B64_sym @g_shared
+    %1:b64 = LD_i64 0, 0, 3, 3, 64, -1, %0, 0 :: (load (s64), addrspace 3)
+    Return
+...
+---
+# When the address has non-memory uses too, memory uses are still folded but
+# the `mov` is kept for the remaining use.
+# CHECK-LABEL: name: fold_partial
+# CHECK: %0:b64 = MOV_B64_sym @g
+# CHECK: %1:b64 = LD_i64 0, 0, 1, 3, 64, -1, @g, 0
+# CHECK: %2:b64 = MULT64rr %0, %0
+name: fold_partial
+tracksRegLiveness: true
+body: |
+  bb.0:
+    %0:b64 = MOV_B64_sym @g
+    %1:b64 = LD_i64 0, 0, 1, 3, 64, -1, %0, 0 :: (load (s64), addrspace 1)
+    %2:b64 = MULT64rr %0, %0
+    Return
+...
diff --git a/llvm/test/CodeGen/RISCV/rv32p.ll b/llvm/test/CodeGen/RISCV/rv32p.ll
index aca224c397ad2..63cdf8ee0ded1 100644
--- a/llvm/test/CodeGen/RISCV/rv32p.ll
+++ b/llvm/test/CodeGen/RISCV/rv32p.ll
@@ -1690,6 +1690,78 @@ define i64 @wsubau_zext_chain_rev(i64 %acc, i32 %a, i32 %b) nounwind {
   ret i64 %sum
 }
 
+; acc + sext(a) -> wadda acc, a, 0
+define i64 @wadda_sext(i64 %acc, i32 %a) nounwind {
+; CHECK-LABEL: wadda_sext:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wadda a0, a2, zero
+; CHECK-NEXT:    ret
+  %ext_a = sext i32 %a to i64
+  %sum = add i64 %acc, %ext_a
+  ret i64 %sum
+}
+
+; sext(a) + acc -> wadda acc, a, 0
+define i64 @wadda_sext_commuted(i64 %acc, i32 %a) nounwind {
+; CHECK-LABEL: wadda_sext_commuted:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wadda a0, a2, zero
+; CHECK-NEXT:    ret
+  %ext_a = sext i32 %a to i64
+  %sum = add i64 %ext_a, %acc
+  ret i64 %sum
+}
+
+; acc + sext(a) + sext(b) -> wadda acc, a, b
+define i64 @wadda_sext_chain(i64 %acc, i32 %a, i32 %b) nounwind {
+; CHECK-LABEL: wadda_sext_chain:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wadda a0, a2, a3
+; CHECK-NEXT:    ret
+  %ext_a = sext i32 %a to i64
+  %ext_b = sext i32 %b to i64
+  %sum1 = add i64 %acc, %ext_a
+  %sum2 = add i64 %sum1, %ext_b
+  ret i64 %sum2
+}
+
+; acc - sext(a) -> wsuba acc, 0, a
+define i64 @wsuba_sext(i64 %acc, i32 %a) nounwind {
+; CHECK-LABEL: wsuba_sext:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wsuba a0, zero, a2
+; CHECK-NEXT:    ret
+  %ext_a = sext i32 %a to i64
+  %sub = sub i64 %acc, %ext_a
+  ret i64 %sub
+}
+
+; (acc + sext(a)) - sext(b) -> wsuba acc, a, b
+define i64 @wsuba_sext_chain(i64 %acc, i32 %a, i32 %b) nounwind {
+; CHECK-LABEL: wsuba_sext_chain:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wsuba a0, a2, a3
+; CHECK-NEXT:    ret
+  %ext_a = sext i32 %a to i64
+  %ext_b = sext i32 %b to i64
+  %sum = add i64 %acc, %ext_a
+  %sub = sub i64 %sum, %ext_b
+  ret i64 %sub
+}
+
+; (acc - sext(a)) + sext(b) -> wsuba acc, b, a
+define i64 @wsuba_sext_chain_rev(i64 %acc, i32 %a, i32 %b) nounwind {
+; CHECK-LABEL: wsuba_sext_chain_rev:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wsuba a0, a3, a2
+; CHECK-NEXT:    ret
+  %ext_a = sext i32 %a to i64
+  %ext_b = sext i32 %b to i64
+  %sub = sub i64 %acc, %ext_a
+  %sum = add i64 %sub, %ext_b
+  ret i64 %sum
+}
+
 define i64 @waddu(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: waddu:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/bmi.ll b/llvm/test/CodeGen/X86/bmi.ll
index bcf1a3eaeb8c2..9e79e2fe95320 100644
--- a/llvm/test/CodeGen/X86/bmi.ll
+++ b/llvm/test/CodeGen/X86/bmi.ll
@@ -2174,3 +2174,52 @@ define i16 @blsi16_trunc(i32 %x) {
   %and = and i16 %t, %neg
   ret i16 %and
 }
+
+define i8 @blsmsk8(i8 %x) nounwind {
+; X86-LABEL: blsmsk8:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    blsmskl %eax, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: blsmsk8:
+; X64:       # %bb.0:
+; X64-NEXT:    blsmskl %edi, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+;
+; EGPR-LABEL: blsmsk8:
+; EGPR:       # %bb.0:
+; EGPR-NEXT:    blsmskl %edi, %eax # encoding: [0xc4,0xe2,0x78,0xf3,0xd7]
+; EGPR-NEXT:    # kill: def $al killed $al killed $eax
+; EGPR-NEXT:    retq # encoding: [0xc3]
+  %y = add i8 %x, -1
+  %z = xor i8 %x, %y
+  ret i8 %z
+}
+
+define i8 @blsmsk8_trunc(i32 %x) nounwind {
+; X86-LABEL: blsmsk8_trunc:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    blsmskl %eax, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: blsmsk8_trunc:
+; X64:       # %bb.0:
+; X64-NEXT:    blsmskl %edi, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+;
+; EGPR-LABEL: blsmsk8_trunc:
+; EGPR:       # %bb.0:
+; EGPR-NEXT:    blsmskl %edi, %eax # encoding: [0xc4,0xe2,0x78,0xf3,0xd7]
+; EGPR-NEXT:    # kill: def $al killed $al killed $eax
+; EGPR-NEXT:    retq # encoding: [0xc3]
+  %t = trunc i32 %x to i8
+  %y = add i8 %t, -1
+  %z = xor i8 %t, %y
+  ret i8 %z
+}
\ No newline at end of file
diff --git a/llvm/test/CodeGen/X86/gfni-xor-fold-avx512.ll b/llvm/test/CodeGen/X86/gfni-xor-fold-avx512.ll
index 1c8a605e3c094..70f5493847ec7 100644
--- a/llvm/test/CodeGen/X86/gfni-xor-fold-avx512.ll
+++ b/llvm/test/CodeGen/X86/gfni-xor-fold-avx512.ll
@@ -229,3 +229,102 @@ define <64 x i8> @test_affine_affine_xor_no_fold_512_different_inputs(<64 x i8>
   %xor = xor <64 x i8> %gfni1, %gfni2
   ret <64 x i8> %xor
 }
+
+define <64 x i8> @test_source_xor_fold_splat2(<64 x i8> %src) nounwind {
+; CHECK-LABEL: test_source_xor_fold_splat2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vgf2p8affineqb $127, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0 # [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    retq
+  %xor = xor <64 x i8> %src, splat(i8 255)
+  %bit2 = call <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8> %xor, <64 x i8> splat(i8 2), i8 128)
+  ret <64 x i8> %bit2
+}
+
+define <64 x i8> @test_source_xor_fold_bitrev(<64 x i8> %src) nounwind {
+; CHECK-LABEL: test_source_xor_fold_bitrev:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vgf2p8affineqb $255, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0 # [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; CHECK-NEXT:    retq
+  %xor = xor <64 x i8> %src, splat(i8 15)
+  %rev = call <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8> %xor, <64 x i8> <i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128>, i8 15)
+  ret <64 x i8> %rev
+}
+
+define <64 x i8> @test_source_xor_fold_rotl2(<64 x i8> %src) nounwind {
+; CHECK-LABEL: test_source_xor_fold_rotl2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vgf2p8affineqb $5, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0 # [32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64]
+; CHECK-NEXT:    retq
+  %xor = xor <64 x i8> %src, splat(i8 193)
+  %rotl2 = call <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8> %xor, <64 x i8> <i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 128, i8 64>, i8 2)
+  ret <64 x i8> %rotl2
+}
+
+define <64 x i8> @test_source_xor_fold_cumulative_parity(<64 x i8> %src) nounwind {
+; CHECK-LABEL: test_source_xor_fold_cumulative_parity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vgf2p8affineqb $158, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0 # [255,127,63,31,15,7,3,1,255,127,63,31,15,7,3,1,255,127,63,31,15,7,3,1,255,127,63,31,15,7,3,1,255,127,63,31,15,7,3,1,255,127,63,31,15,7,3,1,255,127,63,31,15,7,3,1,255,127,63,31,15,7,3,1]
+; CHECK-NEXT:    retq
+  %xor = xor <64 x i8> %src, splat(i8 162)
+  %parity = call <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8> %xor, <64 x i8> <i8 255, i8 127, i8 63, i8 31, i8 15, i8 7, i8 3, i8 1, i8 255, i8 127, i8 63, i8 31, i8 15, i8 7, i8 3, i8 1, i8 255, i8 127, i8 63, i8 31, i8 15, i8 7, i8 3, i8 1, i8 255, i8 127, i8 63, i8 31, i8 15, i8 7, i8 3, i8 1, i8 255, i8 127, i8 63, i8 31, i8 15, i8 7, i8 3, i8 1, i8 255, i8 127, i8 63, i8 31, i8 15, i8 7, i8 3, i8 1, i8 255, i8 127, i8 63, i8 31, i8 15, i8 7, i8 3, i8 1, i8 255, i8 127, i8 63, i8 31, i8 15, i8 7, i8 3, i8 1>, i8 0)
+  ret <64 x i8> %parity
+}
+
+;; Positive test: multi-use still shortens the dependency chain
+define <64 x i8> @test_source_xor_fold_multiuse(<64 x i8> %src, ptr %sink) nounwind {
+; CHECK-LABEL: test_source_xor_fold_multiuse:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vgf2p8affineqb $127, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm1 # [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    vpternlogq {{.*#+}} zmm0 = ~zmm0
+; CHECK-NEXT:    vmovdqa64 %zmm0, (%rdi)
+; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %xor = xor <64 x i8> %src, splat(i8 255)
+  store <64 x i8> %xor, ptr %sink
+  %bit2 = call <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8> %xor, <64 x i8> splat(i8 2), i8 128)
+  ret <64 x i8> %bit2
+}
+
+define <64 x i8> @test_source_xor_no_fold_nonsplat_xor(<64 x i8> %src) nounwind {
+; CHECK-LABEL: test_source_xor_no_fold_nonsplat_xor:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpxorq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; CHECK-NEXT:    vgf2p8affineqb $128, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0 # [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    retq
+  %xor = xor <64 x i8> %src, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32, i8 33, i8 34, i8 35, i8 36, i8 37, i8 38, i8 39, i8 40, i8 41, i8 42, i8 43, i8 44, i8 45, i8 46, i8 47, i8 48, i8 49, i8 50, i8 51, i8 52, i8 53, i8 54, i8 55, i8 56, i8 57, i8 58, i8 59, i8 60, i8 61, i8 62, i8 63>
+  %bit2 = call <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8> %xor, <64 x i8> splat(i8 2), i8 128)
+  ret <64 x i8> %bit2
+}
+
+define <64 x i8> @test_source_xor_no_fold_nonsplat_matrix(<64 x i8> %src) nounwind {
+; CHECK-LABEL: test_source_xor_no_fold_nonsplat_matrix:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpxord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
+; CHECK-NEXT:    vgf2p8affineqb $15, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63]
+; CHECK-NEXT:    retq
+  %xor = xor <64 x i8> %src, splat(i8 15)
+  %gfni = call <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8> %xor, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32, i8 33, i8 34, i8 35, i8 36, i8 37, i8 38, i8 39, i8 40, i8 41, i8 42, i8 43, i8 44, i8 45, i8 46, i8 47, i8 48, i8 49, i8 50, i8 51, i8 52, i8 53, i8 54, i8 55, i8 56, i8 57, i8 58, i8 59, i8 60, i8 61, i8 62, i8 63>, i8 15)
+  ret <64 x i8> %gfni
+}
+
+define <64 x i8> @test_source_xor_no_fold_variable_xor(<64 x i8> %src, <64 x i8> %val) nounwind {
+; CHECK-LABEL: test_source_xor_no_fold_variable_xor:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    vgf2p8affineqb $128, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0 # [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; CHECK-NEXT:    retq
+  %xor = xor <64 x i8> %src, %val
+  %bit2 = call <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8> %xor, <64 x i8> splat(i8 2), i8 128)
+  ret <64 x i8> %bit2
+}
+
+define <64 x i8> @test_source_xor_no_fold_variable_matrix(<64 x i8> %src, <64 x i8> %matrix) nounwind {
+; CHECK-LABEL: test_source_xor_no_fold_variable_matrix:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpxord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
+; CHECK-NEXT:    vgf2p8affineqb $15, %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    retq
+  %xor = xor <64 x i8> %src, splat(i8 15)
+  %gfni = call <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8> %xor, <64 x i8> %matrix, i8 15)
+  ret <64 x i8> %gfni
+}
diff --git a/llvm/test/CodeGen/X86/gfni-xor-fold.ll b/llvm/test/CodeGen/X86/gfni-xor-fold.ll
index 602e317ef2685..429e867e8596b 100644
--- a/llvm/test/CodeGen/X86/gfni-xor-fold.ll
+++ b/llvm/test/CodeGen/X86/gfni-xor-fold.ll
@@ -444,3 +444,154 @@ define <32 x i8> @test_affine_affine_same_matrix_fold_256(<32 x i8> %src1, <32 x
   %xor = xor <32 x i8> %gfni1, %gfni2
   ret <32 x i8> %xor
 }
+
+define <16 x i8> @test_source_xor_fold_splat2(<16 x i8> %src) nounwind {
+; AVX-LABEL: test_source_xor_fold_splat2:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vgf2p8affineqb $127, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_source_xor_fold_splat2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vgf2p8affineqb $127, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0 # [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; AVX512-NEXT:    retq
+  %xor = xor <16 x i8> %src, splat(i8 255)
+  %bit2 = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %xor, <16 x i8> splat(i8 2), i8 128)
+  ret <16 x i8> %bit2
+}
+
+define <16 x i8> @test_source_xor_fold_bitrev(<16 x i8> %src) nounwind {
+; AVX-LABEL: test_source_xor_fold_bitrev:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vgf2p8affineqb $255, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_source_xor_fold_bitrev:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vgf2p8affineqb $255, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0 # [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; AVX512-NEXT:    retq
+  %xor = xor <16 x i8> %src, splat(i8 15)
+  %rev = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %xor, <16 x i8> <i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128>, i8 15)
+  ret <16 x i8> %rev
+}
+
+define <16 x i8> @test_source_xor_fold_rotl2(<16 x i8> %src) nounwind {
+; AVX-LABEL: test_source_xor_fold_rotl2:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vgf2p8affineqb $5, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_source_xor_fold_rotl2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vgf2p8affineqb $5, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0 # [32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64]
+; AVX512-NEXT:    retq
+  %xor = xor <16 x i8> %src, splat(i8 193)
+  %rotl2 = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %xor, <16 x i8> <i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 128, i8 64, i8 32, i8 16, i8 8, i8 4, i8 2, i8 1, i8 128, i8 64>, i8 2)
+  ret <16 x i8> %rotl2
+}
+
+define <16 x i8> @test_source_xor_fold_cumulative_parity(<16 x i8> %src) nounwind {
+; AVX-LABEL: test_source_xor_fold_cumulative_parity:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vgf2p8affineqb $158, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [255,127,63,31,15,7,3,1,255,127,63,31,15,7,3,1]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_source_xor_fold_cumulative_parity:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vgf2p8affineqb $158, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0 # [255,127,63,31,15,7,3,1,255,127,63,31,15,7,3,1]
+; AVX512-NEXT:    retq
+  %xor = xor <16 x i8> %src, splat(i8 162)
+  %parity = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %xor, <16 x i8> <i8 255, i8 127, i8 63, i8 31, i8 15, i8 7, i8 3, i8 1, i8 255, i8 127, i8 63, i8 31, i8 15, i8 7, i8 3, i8 1>, i8 0)
+  ret <16 x i8> %parity
+}
+
+;; Positive test: multi-use still shortens the dependency chain
+define <16 x i8> @test_source_xor_fold_multiuse(<16 x i8> %src, ptr %sink) nounwind {
+; AVX-LABEL: test_source_xor_fold_multiuse:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm1
+; AVX-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX-NEXT:    vgf2p8affineqb $127, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_source_xor_fold_multiuse:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vgf2p8affineqb $127, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm1 # [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; AVX512-NEXT:    vpternlogq {{.*#+}} xmm0 = ~xmm0
+; AVX512-NEXT:    vmovdqa %xmm0, (%rdi)
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512-NEXT:    retq
+  %xor = xor <16 x i8> %src, splat(i8 255)
+  store <16 x i8> %xor, ptr %sink
+  %bit2 = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %xor, <16 x i8> splat(i8 2), i8 128)
+  ret <16 x i8> %bit2
+}
+
+define <16 x i8> @test_source_xor_no_fold_nonsplat_xor(<16 x i8> %src) nounwind {
+; AVX-LABEL: test_source_xor_no_fold_nonsplat_xor:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vgf2p8affineqb $128, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_source_xor_no_fold_nonsplat_xor:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vgf2p8affineqb $128, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0 # [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; AVX512-NEXT:    retq
+  %xor = xor <16 x i8> %src, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>
+  %bit2 = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %xor, <16 x i8> splat(i8 2), i8 128)
+  ret <16 x i8> %bit2
+}
+
+define <16 x i8> @test_source_xor_no_fold_nonsplat_matrix(<16 x i8> %src) nounwind {
+; AVX-LABEL: test_source_xor_no_fold_nonsplat_matrix:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vgf2p8affineqb $15, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_source_xor_no_fold_nonsplat_matrix:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
+; AVX512-NEXT:    vgf2p8affineqb $15, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX512-NEXT:    retq
+  %xor = xor <16 x i8> %src, splat(i8 15)
+  %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %xor, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, i8 15)
+  ret <16 x i8> %gfni
+}
+
+define <16 x i8> @test_source_xor_no_fold_variable_xor(<16 x i8> %src, <16 x i8> %val) nounwind {
+; AVX-LABEL: test_source_xor_no_fold_variable_xor:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vgf2p8affineqb $128, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_source_xor_no_fold_variable_xor:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vgf2p8affineqb $128, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0 # [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; AVX512-NEXT:    retq
+  %xor = xor <16 x i8> %src, %val
+  %bit2 = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %xor, <16 x i8> splat(i8 2), i8 128)
+  ret <16 x i8> %bit2
+}
+
+define <16 x i8> @test_source_xor_no_fold_variable_matrix(<16 x i8> %src, <16 x i8> %matrix) nounwind {
+; AVX-LABEL: test_source_xor_no_fold_variable_matrix:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vgf2p8affineqb $15, %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_source_xor_no_fold_variable_matrix:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
+; AVX512-NEXT:    vgf2p8affineqb $15, %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+  %xor = xor <16 x i8> %src, splat(i8 15)
+  %gfni = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %xor, <16 x i8> %matrix, i8 15)
+  ret <16 x i8> %gfni
+}
diff --git a/llvm/test/CodeGen/X86/vec_minmax_match.ll b/llvm/test/CodeGen/X86/vec_minmax_match.ll
index 56a45fa9a4ff5..6f97937c95b39 100644
--- a/llvm/test/CodeGen/X86/vec_minmax_match.ll
+++ b/llvm/test/CodeGen/X86/vec_minmax_match.ll
@@ -7,26 +7,26 @@
 define <4 x i32> @smin_vec1(<4 x i32> %x) {
 ; CHECK-LABEL: smin_vec1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; CHECK-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %not_x = xor <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
-  %cmp = icmp sgt <4 x i32> %x, zeroinitializer
-  %sel = select <4 x i1> %cmp, <4 x i32> %not_x, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
+  %1 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %x, <4 x i32> zeroinitializer)
+  %sel = xor <4 x i32> %1, splat (i32 -1)
   ret <4 x i32> %sel
 }
 
 define <4 x i32> @smin_vec2(<4 x i32> %x) {
 ; CHECK-LABEL: smin_vec2:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; CHECK-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %not_x = xor <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
-  %cmp = icmp slt <4 x i32> %x, zeroinitializer
-  %sel = select <4 x i1> %cmp, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> %not_x
+  %1 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %x, <4 x i32> zeroinitializer)
+  %sel = xor <4 x i32> %1, splat (i32 -1)
   ret <4 x i32> %sel
 }
 
@@ -121,8 +121,7 @@ define <4 x i32> @umax_vec1(<4 x i32> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpmaxud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %cmp = icmp slt <4 x i32> %x, zeroinitializer
-  %sel = select <4 x i1> %cmp, <4 x i32> %x, <4 x i32> <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+  %sel = call <4 x i32> @llvm.umax.v4i32(<4 x i32> %x, <4 x i32> splat (i32 2147483647))
   ret <4 x i32> %sel
 }
 
@@ -131,8 +130,7 @@ define <4 x i32> @umax_vec2(<4 x i32> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpmaxud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %cmp = icmp sgt <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
-  %sel = select <4 x i1> %cmp, <4 x i32> <i32 2147483648, i32 2147483648, i32 2147483648, i32 2147483648>, <4 x i32> %x
+  %sel = call <4 x i32> @llvm.umax.v4i32(<4 x i32> %x, <4 x i32> splat (i32 -2147483648))
   ret <4 x i32> %sel
 }
 
@@ -141,8 +139,7 @@ define <4 x i32> @umin_vec1(<4 x i32> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %cmp = icmp slt <4 x i32> %x, zeroinitializer
-  %sel = select <4 x i1> %cmp, <4 x i32> <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>, <4 x i32> %x
+  %sel = call <4 x i32> @llvm.umin.v4i32(<4 x i32> %x, <4 x i32> splat (i32 2147483647))
   ret <4 x i32> %sel
 }
 
@@ -151,8 +148,7 @@ define <4 x i32> @umin_vec2(<4 x i32> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %cmp = icmp sgt <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
-  %sel = select <4 x i1> %cmp, <4 x i32> %x, <4 x i32> <i32 2147483648, i32 2147483648, i32 2147483648, i32 2147483648>
+  %sel = call <4 x i32> @llvm.umin.v4i32(<4 x i32> %x, <4 x i32> splat (i32 -2147483648))
   ret <4 x i32> %sel
 }
 
@@ -164,13 +160,11 @@ define <4 x i32> @umin_vec2(<4 x i32> %x) {
 define <4 x i32> @clamp_signed1(<4 x i32> %x) {
 ; CHECK-LABEL: clamp_signed1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpminsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    vpmaxsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT:    vpminsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %cmp2 = icmp slt <4 x i32> %x, <i32 255, i32 255, i32 255, i32 255>
-  %min = select <4 x i1> %cmp2, <4 x i32> %x, <4 x i32><i32 255, i32 255, i32 255, i32 255>
-  %cmp1 = icmp slt <4 x i32> %x, <i32 15, i32 15, i32 15, i32 15>
-  %r = select <4 x i1> %cmp1, <4 x i32><i32 15, i32 15, i32 15, i32 15>, <4 x i32> %min
+  %1 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %x, <4 x i32> splat (i32 15))
+  %r = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %1, <4 x i32> splat (i32 255))
   ret <4 x i32> %r
 }
 
@@ -182,10 +176,8 @@ define <4 x i32> @clamp_signed2(<4 x i32> %x) {
 ; CHECK-NEXT:    vpmaxsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    vpminsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %cmp2 = icmp sgt <4 x i32> %x, <i32 15, i32 15, i32 15, i32 15>
-  %max = select <4 x i1> %cmp2, <4 x i32> %x, <4 x i32><i32 15, i32 15, i32 15, i32 15>
-  %cmp1 = icmp sgt <4 x i32> %x, <i32 255, i32 255, i32 255, i32 255>
-  %r = select <4 x i1> %cmp1, <4 x i32><i32 255, i32 255, i32 255, i32 255>, <4 x i32> %max
+  %max = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %x, <4 x i32> splat (i32 15))
+  %r = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %max, <4 x i32> splat (i32 255))
   ret <4 x i32> %r
 }
 
@@ -194,13 +186,11 @@ define <4 x i32> @clamp_signed2(<4 x i32> %x) {
 define <4 x i32> @clamp_unsigned1(<4 x i32> %x) {
 ; CHECK-LABEL: clamp_unsigned1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    vpmaxud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %cmp2 = icmp ult <4 x i32> %x, <i32 255, i32 255, i32 255, i32 255>
-  %min = select <4 x i1> %cmp2, <4 x i32> %x, <4 x i32><i32 255, i32 255, i32 255, i32 255>
-  %cmp1 = icmp ult <4 x i32> %x, <i32 15, i32 15, i32 15, i32 15>
-  %r = select <4 x i1> %cmp1, <4 x i32><i32 15, i32 15, i32 15, i32 15>, <4 x i32> %min
+  %1 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> %x, <4 x i32> splat (i32 15))
+  %r = call <4 x i32> @llvm.umin.v4i32(<4 x i32> %1, <4 x i32> splat (i32 255))
   ret <4 x i32> %r
 }
 
@@ -212,37 +202,19 @@ define <4 x i32> @clamp_unsigned2(<4 x i32> %x) {
 ; CHECK-NEXT:    vpmaxud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %cmp2 = icmp ugt <4 x i32> %x, <i32 15, i32 15, i32 15, i32 15>
-  %max = select <4 x i1> %cmp2, <4 x i32> %x, <4 x i32><i32 15, i32 15, i32 15, i32 15>
-  %cmp1 = icmp ugt <4 x i32> %x, <i32 255, i32 255, i32 255, i32 255>
-  %r = select <4 x i1> %cmp1, <4 x i32><i32 255, i32 255, i32 255, i32 255>, <4 x i32> %max
+  %max = call <4 x i32> @llvm.umax.v4i32(<4 x i32> %x, <4 x i32> splat (i32 15))
+  %r = call <4 x i32> @llvm.umin.v4i32(<4 x i32> %max, <4 x i32> splat (i32 255))
   ret <4 x i32> %r
 }
 
 define <4 x i32> @umin_not_ops(<4 x i32> %x) {
 ; CHECK-LABEL: umin_not_ops:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmaxud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; CHECK-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %not_x = xor <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
-  %cmp = icmp ugt <4 x i32> %x, <i32 4, i32 4, i32 4, i32 4>
-  %sel = select <4 x i1> %cmp, <4 x i32> %not_x, <4 x i32> <i32 -5, i32 -5, i32 -5, i32 -5>
+  %1 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> %x, <4 x i32> splat (i32 4))
+  %sel = xor <4 x i32> %1, splat (i32 -1)
   ret <4 x i32> %sel
 }
-
-define <4 x i32> @wrong_pred_for_smin_with_subnsw(<4 x i32> %x, <4 x i32> %y) {
-; CHECK-LABEL: wrong_pred_for_smin_with_subnsw:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpsubd %xmm1, %xmm0, %xmm2
-; CHECK-NEXT:    vpminud %xmm1, %xmm0, %xmm1
-; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-  %sub = sub nsw <4 x i32> %x, %y
-  %cmp = icmp ugt <4 x i32> %x, %y
-  %sel = select <4 x i1> %cmp, <4 x i32> zeroinitializer, <4 x i32> %sub
-  ret <4 x i32> %sel
-}
-
diff --git a/llvm/test/Transforms/ArgumentPromotion/noipa.ll b/llvm/test/Transforms/ArgumentPromotion/noipa.ll
new file mode 100644
index 0000000000000..a70e8d0aa0ecc
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/noipa.ll
@@ -0,0 +1,27 @@
+; RUN: opt -passes=argpromotion -S < %s | FileCheck %s
+
+; ArgumentPromotion promoting a by-ref argument of a `noipa` function.
+; If `noipa` is working, argpromotion should NOT promote the pointer argument,
+; because it requires inspecting the function body to determine load patterns.
+
+; CHECK: define internal i32 @foo(i32 %p.0.val)
+define internal i32 @foo(ptr %p) noinline {
+  %v = load i32, ptr %p
+  ret i32 %v
+}
+
+; CHECK: define internal i32 @foo_noipa(ptr %p)
+define internal i32 @foo_noipa(ptr %p) noipa noinline {
+  %v = load i32, ptr %p
+  ret i32 %v
+}
+
+define i32 @bar() {
+  %a = alloca i32
+  store i32 7, ptr %a
+  %r = call i32 @foo(ptr %a)
+  %b = alloca i32
+  store i32 %r, ptr %b
+  %s = call i32 @foo_noipa(ptr %b)
+  ret i32 %s
+}
diff --git a/llvm/test/Transforms/DeadArgElim/noipa.ll b/llvm/test/Transforms/DeadArgElim/noipa.ll
new file mode 100644
index 0000000000000..2834f5fed4ccb
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/noipa.ll
@@ -0,0 +1,23 @@
+; RUN: opt -passes=deadargelim -S < %s | FileCheck %s
+
+; DeadArgumentElimination removing unused args of a `noipa` function.
+; If `noipa` is working, DAE should NOT remove the unused argument %y from @foo,
+; because determining which arguments are dead requires inspecting the
+; definition.
+
+; CHECK: define internal i32 @foo(i32 %x)
+define internal i32 @foo(i32 %x, i32 %y) noinline {
+  ret i32 %x
+}
+
+; CHECK: define internal i32 @foo_noipa(i32 %x, i32 %y)
+define internal i32 @foo_noipa(i32 %x, i32 %y) noipa noinline {
+  ret i32 %x
+}
+
+define i32 @bar() {
+  %r = call i32 @foo(i32 1, i32 2)
+  %s = call i32 @foo_noipa(i32 1, i32 2)
+  %res = add i32 %r, %s
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/FunctionSpecialization/noipa.ll b/llvm/test/Transforms/FunctionSpecialization/noipa.ll
new file mode 100644
index 0000000000000..e97247a16defb
--- /dev/null
+++ b/llvm/test/Transforms/FunctionSpecialization/noipa.ll
@@ -0,0 +1,24 @@
+; RUN: opt -passes='ipsccp<func-spec>' -force-specialization -S < %s | FileCheck %s
+
+; Function specialisation should not activate for `noipa` functions.
+
+define i32 @foo(i32 %x, i32 %y) {
+  %r2 = add i32 %x, %y
+  ret i32 %r2
+}
+
+define i32 @foo_noipa(i32 %x, i32 %y) noipa {
+  %r2 = add i32 %x, %y
+  ret i32 %r2
+}
+
+; CHECK:      call i32 @foo.specialized.1(
+; CHECK:      call i32 @foo_noipa(
+; CHECK:      define internal i32 @foo.specialized.1(
+; CHECK-NOT:  define internal i32 @foo_noipa.specialized.1(
+define i32 @test() {
+  %r = call i32 @foo(i32 0, i32 5)
+  %s = call i32 @foo_noipa(i32 0, i32 5)
+  %res = add i32 %r, %s
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/GlobalOpt/global-constructor-noipa.ll b/llvm/test/Transforms/GlobalOpt/global-constructor-noipa.ll
new file mode 100644
index 0000000000000..00caa3595af27
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/global-constructor-noipa.ll
@@ -0,0 +1,36 @@
+; RUN: opt -passes=globalopt -S -o - < %s | FileCheck %s
+
+; Calls to `noipa` functions in constructors prevent evaluation.
+
+; CHECK: @object = local_unnamed_addr global i32 4
+@object = local_unnamed_addr global i32 0
+
+define void @ctor() {
+  store i32 4, ptr @object
+  %a = bitcast ptr @object to ptr
+  call void @test(ptr %a)
+  ret void
+}
+
+define void @test(ptr %ptr) {
+  ret void
+}
+
+; CHECK: @object_noipa = local_unnamed_addr global i32 0
+@object_noipa = local_unnamed_addr global i32 0
+
+define void @ctor_noipa() {
+  store i32 4, ptr @object_noipa
+  %a = bitcast ptr @object_noipa to ptr
+  call void @test_noipa(ptr %a)
+  ret void
+}
+
+define void @test_noipa(ptr %ptr) noipa {
+  ret void
+}
+
+@llvm.global_ctors = appending constant
+  [2 x { i32, ptr, ptr }]
+  [ { i32, ptr, ptr } { i32 65535, ptr @ctor, ptr null },
+    { i32, ptr, ptr } { i32 65535, ptr @ctor_noipa, ptr null } ]
diff --git a/llvm/test/Transforms/GlobalOpt/noipa.ll b/llvm/test/Transforms/GlobalOpt/noipa.ll
new file mode 100644
index 0000000000000..16046ca70112a
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/noipa.ll
@@ -0,0 +1,22 @@
+; RUN: opt -passes=globalopt -S < %s | FileCheck %s
+
+; GlobalOpt changing calling convention of a `noipa` function.
+; If `noipa` is working, GlobalOpt should NOT change the CC of @foo,
+; because it requires analyzing callers/callees interprocedurally.
+
+; CHECK: define internal fastcc i32 @foo(
+define internal i32 @foo(i32 %x) noinline {
+  ret i32 %x
+}
+
+; CHECK: define internal i32 @foo_noipa(
+define internal i32 @foo_noipa(i32 %x) noipa noinline {
+  ret i32 %x
+}
+
+define i32 @bar() {
+  %r = call i32 @foo(i32 5)
+  %s = call i32 @foo_noipa(i32 5)
+  %res = add i32 %r, %s
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/GlobalOpt/resolve-fmv-ifunc-noipa.ll b/llvm/test/Transforms/GlobalOpt/resolve-fmv-ifunc-noipa.ll
new file mode 100644
index 0000000000000..6b373eef7977e
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/resolve-fmv-ifunc-noipa.ll
@@ -0,0 +1,76 @@
+; REQUIRES: aarch64-registered-target
+
+; RUN: opt --passes=globalopt -o - -S < %s | FileCheck %s
+
+; `noipa` resolvers are not eligible for inspection
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-linux-gnu"
+
+$callee.resolver = comdat any
+$caller.resolver = comdat any
+
+@__aarch64_cpu_features = external local_unnamed_addr global { i64 }
+
+@callee = weak_odr ifunc i32 (), ptr @callee.resolver
+@caller = weak_odr ifunc i32 (), ptr @caller.resolver
+
+declare void @__init_cpu_features_resolver() local_unnamed_addr
+
+declare i32 @callee.default() #0
+declare i32 @callee._Msve() #1
+declare i32 @callee._Msve2() #2
+
+define weak_odr ptr @callee.resolver() noipa comdat {
+resolver_entry:
+  tail call void @__init_cpu_features_resolver()
+  %0 = load i64, ptr @__aarch64_cpu_features, align 8
+  %1 = and i64 %0, 69793284352
+  %2 = icmp eq i64 %1, 69793284352
+  %3 = and i64 %0, 1073807616
+  %4 = icmp eq i64 %3, 1073807616
+  %callee._Msve.callee.default = select i1 %4, ptr @callee._Msve, ptr @callee.default
+  %common.ret.op = select i1 %2, ptr @callee._Msve2, ptr %callee._Msve.callee.default
+  ret ptr %common.ret.op
+}
+
+; CHECK: define i32 @caller._Msve()
+; CHECK: tail call i32 @callee()
+define i32 @caller._Msve() #1 {
+entry:
+  %call = tail call i32 @callee()
+  ret i32 %call
+}
+
+; CHECK: define i32 @caller._Msve2()
+; CHECK: tail call i32 @callee()
+define i32 @caller._Msve2() #2 {
+entry:
+  %call = tail call i32 @callee()
+  ret i32 %call
+}
+
+; CHECK: define i32 @caller.default()
+; CHECK: tail call i32 @callee()
+define i32 @caller.default() #0 {
+entry:
+  %call = tail call i32 @callee()
+  ret i32 %call
+}
+
+define weak_odr ptr @caller.resolver() comdat {
+resolver_entry:
+  tail call void @__init_cpu_features_resolver()
+  %0 = load i64, ptr @__aarch64_cpu_features, align 8
+  %1 = and i64 %0, 69793284352
+  %2 = icmp eq i64 %1, 69793284352
+  %3 = and i64 %0, 1073807616
+  %4 = icmp eq i64 %3, 1073807616
+  %caller._Msve.caller.default = select i1 %4, ptr @caller._Msve, ptr @caller.default
+  %common.ret.op = select i1 %2, ptr @caller._Msve2, ptr %caller._Msve.caller.default
+  ret ptr %common.ret.op
+}
+
+attributes #0 = { "fmv-features" }
+attributes #1 = { "fmv-features"="sve" }
+attributes #2 = { "fmv-features"="sve2" }
diff --git a/llvm/test/Transforms/GlobalOpt/resolve-static-ifunc-noipa.ll b/llvm/test/Transforms/GlobalOpt/resolve-static-ifunc-noipa.ll
new file mode 100644
index 0000000000000..3b899e6f3a344
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/resolve-static-ifunc-noipa.ll
@@ -0,0 +1,29 @@
+; RUN: opt --passes=globalopt -o - -S < %s | FileCheck %s
+
+; `noipa` resolvers are not eligible for inspection
+
+@static.ifunc = internal ifunc void (), ptr @static.resolver
+
+define ptr @static.resolver() {
+  ret ptr @static._Msimd
+}
+define void @static._Msimd() {
+  ret void
+}
+define void @static.default() {
+  ret void
+}
+
+@static_noipa.ifunc = internal ifunc void (), ptr @static_noipa.resolver
+
+define ptr @static_noipa.resolver() noipa {
+  ret ptr @static._Msimd
+}
+
+define void @caller() {
+  ; CHECK: call void @static._Msimd()
+  call void @static.ifunc()
+  ; CHECK: call void @static_noipa.ifunc()
+  call void @static_noipa.ifunc()
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/noipa.ll b/llvm/test/Transforms/Inline/noipa.ll
new file mode 100644
index 0000000000000..d292fcb99bd09
--- /dev/null
+++ b/llvm/test/Transforms/Inline/noipa.ll
@@ -0,0 +1,23 @@
+; RUN: opt -passes=inline -S < %s | FileCheck %s
+
+; Ensure `noipa` does _not_ control inlining. The `noipa` attribute controls
+; _other_ interprocedural optimisations by affecting the result of
+; `isInterposable` and `isDefinitionExact` checks, but inlining is
+; controlled separately by the `noinline` attribute.
+
+define internal i32 @noipa(i32 %x) noipa {
+  ret i32 %x
+}
+
+define internal i32 @noipa_noinline(i32 %x) noipa noinline {
+  ret i32 %x
+}
+
+define i32 @bob() {
+  %a = call i32 @noipa(i32 1)
+  %b = call i32 @noipa_noinline(i32 %a)
+  ret i32 %b
+}
+
+; CHECK-NOT: call i32 @noipa(
+; CHECK: call i32 @noipa_noinline(
diff --git a/llvm/test/Transforms/InstSimplify/maxmin.ll b/llvm/test/Transforms/InstSimplify/maxmin.ll
index 048850796744d..9b8da488e3fbc 100644
--- a/llvm/test/Transforms/InstSimplify/maxmin.ll
+++ b/llvm/test/Transforms/InstSimplify/maxmin.ll
@@ -75,7 +75,10 @@ define i1 @max1(i32 %x, i32 %y) {
 
 define i1 @max2(i32 %x, i32 %y) {
 ; CHECK-LABEL: @max2(
-; CHECK-NEXT:    ret i1 true
+; CHECK-NEXT:    [[C:%.*]] = icmp sge i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sge i32 [[M]], [[X]]
+; CHECK-NEXT:    ret i1 [[R]]
 ;
   %c = icmp sge i32 %x, %y
   %m = select i1 %c, i32 %x, i32 %y
@@ -95,7 +98,10 @@ define i1 @max3(i32 %x, i32 %y) {
 
 define i1 @max4(i32 %x, i32 %y) {
 ; CHECK-LABEL: @max4(
-; CHECK-NEXT:    ret i1 true
+; CHECK-NEXT:    [[C:%.*]] = icmp uge i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i32 [[M]], [[X]]
+; CHECK-NEXT:    ret i1 [[R]]
 ;
   %c = icmp uge i32 %x, %y
   %m = select i1 %c, i32 %x, i32 %y
@@ -115,7 +121,10 @@ define i1 @max5(i32 %x, i32 %y) {
 
 define i1 @max6(i32 %x, i32 %y) {
 ; CHECK-LABEL: @max6(
-; CHECK-NEXT:    ret i1 true
+; CHECK-NEXT:    [[C:%.*]] = icmp sge i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sle i32 [[X]], [[M]]
+; CHECK-NEXT:    ret i1 [[R]]
 ;
   %c = icmp sge i32 %x, %y
   %m = select i1 %c, i32 %x, i32 %y
@@ -135,7 +144,10 @@ define i1 @max7(i32 %x, i32 %y) {
 
 define i1 @max8(i32 %x, i32 %y) {
 ; CHECK-LABEL: @max8(
-; CHECK-NEXT:    ret i1 true
+; CHECK-NEXT:    [[C:%.*]] = icmp uge i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i32 [[X]], [[M]]
+; CHECK-NEXT:    ret i1 [[R]]
 ;
   %c = icmp uge i32 %x, %y
   %m = select i1 %c, i32 %x, i32 %y
@@ -145,7 +157,10 @@ define i1 @max8(i32 %x, i32 %y) {
 
 define i1 @min1(i32 %x, i32 %y) {
 ; CHECK-LABEL: @min1(
-; CHECK-NEXT:    ret i1 false
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[C]], i32 [[Y]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i32 [[M]], [[X]]
+; CHECK-NEXT:    ret i1 [[R]]
 ;
   %c = icmp sgt i32 %x, %y
   %m = select i1 %c, i32 %y, i32 %x
@@ -165,7 +180,10 @@ define i1 @min2(i32 %x, i32 %y) {
 
 define i1 @min3(i32 %x, i32 %y) {
 ; CHECK-LABEL: @min3(
-; CHECK-NEXT:    ret i1 false
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[C]], i32 [[Y]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i32 [[M]], [[X]]
+; CHECK-NEXT:    ret i1 [[R]]
 ;
   %c = icmp ugt i32 %x, %y
   %m = select i1 %c, i32 %y, i32 %x
@@ -185,7 +203,10 @@ define i1 @min4(i32 %x, i32 %y) {
 
 define i1 @min5(i32 %x, i32 %y) {
 ; CHECK-LABEL: @min5(
-; CHECK-NEXT:    ret i1 false
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[C]], i32 [[Y]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i32 [[X]], [[M]]
+; CHECK-NEXT:    ret i1 [[R]]
 ;
   %c = icmp sgt i32 %x, %y
   %m = select i1 %c, i32 %y, i32 %x
@@ -205,7 +226,10 @@ define i1 @min6(i32 %x, i32 %y) {
 
 define i1 @min7(i32 %x, i32 %y) {
 ; CHECK-LABEL: @min7(
-; CHECK-NEXT:    ret i1 false
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[C]], i32 [[Y]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i32 [[X]], [[M]]
+; CHECK-NEXT:    ret i1 [[R]]
 ;
   %c = icmp ugt i32 %x, %y
   %m = select i1 %c, i32 %y, i32 %x
@@ -225,7 +249,12 @@ define i1 @min8(i32 %x, i32 %y) {
 
 define i1 @maxmin1(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @maxmin1(
-; CHECK-NEXT:    ret i1 true
+; CHECK-NEXT:    [[C1:%.*]] = icmp sge i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[C1]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp sge i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[C2]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    [[C:%.*]] = icmp sge i32 [[MAX]], [[MIN]]
+; CHECK-NEXT:    ret i1 [[C]]
 ;
   %c1 = icmp sge i32 %x, %y
   %max = select i1 %c1, i32 %x, i32 %y
@@ -237,7 +266,12 @@ define i1 @maxmin1(i32 %x, i32 %y, i32 %z) {
 
 define i1 @maxmin2(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @maxmin2(
-; CHECK-NEXT:    ret i1 false
+; CHECK-NEXT:    [[C1:%.*]] = icmp sge i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[C1]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp sge i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[C2]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[MIN]], [[MAX]]
+; CHECK-NEXT:    ret i1 [[C]]
 ;
   %c1 = icmp sge i32 %x, %y
   %max = select i1 %c1, i32 %x, i32 %y
@@ -249,7 +283,12 @@ define i1 @maxmin2(i32 %x, i32 %y, i32 %z) {
 
 define i1 @maxmin3(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @maxmin3(
-; CHECK-NEXT:    ret i1 true
+; CHECK-NEXT:    [[C1:%.*]] = icmp sge i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[C1]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp sge i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[C2]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    [[C:%.*]] = icmp sle i32 [[MIN]], [[MAX]]
+; CHECK-NEXT:    ret i1 [[C]]
 ;
   %c1 = icmp sge i32 %x, %y
   %max = select i1 %c1, i32 %x, i32 %y
@@ -261,7 +300,12 @@ define i1 @maxmin3(i32 %x, i32 %y, i32 %z) {
 
 define i1 @maxmin4(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @maxmin4(
-; CHECK-NEXT:    ret i1 false
+; CHECK-NEXT:    [[C1:%.*]] = icmp sge i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[C1]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp sge i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[C2]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[MAX]], [[MIN]]
+; CHECK-NEXT:    ret i1 [[C]]
 ;
   %c1 = icmp sge i32 %x, %y
   %max = select i1 %c1, i32 %x, i32 %y
@@ -273,7 +317,12 @@ define i1 @maxmin4(i32 %x, i32 %y, i32 %z) {
 
 define i1 @maxmin5(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @maxmin5(
-; CHECK-NEXT:    ret i1 true
+; CHECK-NEXT:    [[C1:%.*]] = icmp uge i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[C1]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp uge i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[C2]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    [[C:%.*]] = icmp uge i32 [[MAX]], [[MIN]]
+; CHECK-NEXT:    ret i1 [[C]]
 ;
   %c1 = icmp uge i32 %x, %y
   %max = select i1 %c1, i32 %x, i32 %y
@@ -285,7 +334,12 @@ define i1 @maxmin5(i32 %x, i32 %y, i32 %z) {
 
 define i1 @maxmin6(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @maxmin6(
-; CHECK-NEXT:    ret i1 false
+; CHECK-NEXT:    [[C1:%.*]] = icmp uge i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[C1]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp uge i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[C2]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[MIN]], [[MAX]]
+; CHECK-NEXT:    ret i1 [[C]]
 ;
   %c1 = icmp uge i32 %x, %y
   %max = select i1 %c1, i32 %x, i32 %y
@@ -297,7 +351,12 @@ define i1 @maxmin6(i32 %x, i32 %y, i32 %z) {
 
 define i1 @maxmin7(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @maxmin7(
-; CHECK-NEXT:    ret i1 true
+; CHECK-NEXT:    [[C1:%.*]] = icmp uge i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[C1]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp uge i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[C2]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ule i32 [[MIN]], [[MAX]]
+; CHECK-NEXT:    ret i1 [[C]]
 ;
   %c1 = icmp uge i32 %x, %y
   %max = select i1 %c1, i32 %x, i32 %y
@@ -309,7 +368,12 @@ define i1 @maxmin7(i32 %x, i32 %y, i32 %z) {
 
 define i1 @maxmin8(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @maxmin8(
-; CHECK-NEXT:    ret i1 false
+; CHECK-NEXT:    [[C1:%.*]] = icmp uge i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[C1]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp uge i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[C2]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[MAX]], [[MIN]]
+; CHECK-NEXT:    ret i1 [[C]]
 ;
   %c1 = icmp uge i32 %x, %y
   %max = select i1 %c1, i32 %x, i32 %y
@@ -322,7 +386,9 @@ define i1 @maxmin8(i32 %x, i32 %y, i32 %z) {
 define i1 @eqcmp1(i32 %x, i32 %y) {
 ; CHECK-LABEL: @eqcmp1(
 ; CHECK-NEXT:    [[C:%.*]] = icmp sge i32 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    ret i1 [[C]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 [[MAX]], [[X]]
+; CHECK-NEXT:    ret i1 [[R]]
 ;
   %c = icmp sge i32 %x, %y
   %max = select i1 %c, i32 %x, i32 %y
@@ -333,7 +399,9 @@ define i1 @eqcmp1(i32 %x, i32 %y) {
 define i1 @eqcmp2(i32 %x, i32 %y) {
 ; CHECK-LABEL: @eqcmp2(
 ; CHECK-NEXT:    [[C:%.*]] = icmp sge i32 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    ret i1 [[C]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 [[X]], [[MAX]]
+; CHECK-NEXT:    ret i1 [[R]]
 ;
   %c = icmp sge i32 %x, %y
   %max = select i1 %c, i32 %x, i32 %y
@@ -344,7 +412,9 @@ define i1 @eqcmp2(i32 %x, i32 %y) {
 define i1 @eqcmp3(i32 %x, i32 %y) {
 ; CHECK-LABEL: @eqcmp3(
 ; CHECK-NEXT:    [[C:%.*]] = icmp uge i32 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    ret i1 [[C]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 [[MAX]], [[X]]
+; CHECK-NEXT:    ret i1 [[R]]
 ;
   %c = icmp uge i32 %x, %y
   %max = select i1 %c, i32 %x, i32 %y
@@ -355,7 +425,9 @@ define i1 @eqcmp3(i32 %x, i32 %y) {
 define i1 @eqcmp4(i32 %x, i32 %y) {
 ; CHECK-LABEL: @eqcmp4(
 ; CHECK-NEXT:    [[C:%.*]] = icmp uge i32 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    ret i1 [[C]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 [[X]], [[MAX]]
+; CHECK-NEXT:    ret i1 [[R]]
 ;
   %c = icmp uge i32 %x, %y
   %max = select i1 %c, i32 %x, i32 %y
diff --git a/llvm/test/Transforms/LoopInterchange/reductions-non-wrapped-operations.ll b/llvm/test/Transforms/LoopInterchange/reductions-non-wrapped-operations.ll
index 29067ff926347..3f89019e9c23f 100644
--- a/llvm/test/Transforms/LoopInterchange/reductions-non-wrapped-operations.ll
+++ b/llvm/test/Transforms/LoopInterchange/reductions-non-wrapped-operations.ll
@@ -241,8 +241,7 @@ define void @reduction_smin(ptr %A, i32 %init) {
 ; CHECK:       [[FOR_J_SPLIT1]]:
 ; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds [2 x [2 x i32]], ptr [[A]], i32 0, i32 [[J]], i32 [[I]]
 ; CHECK-NEXT:    [[A:%.*]] = load i32, ptr [[IDX]], align 4
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[A]], [[SMIN_J]]
-; CHECK-NEXT:    [[SMIN_J_NEXT]] = select i1 [[CMP]], i32 [[A]], i32 [[SMIN_J]]
+; CHECK-NEXT:    [[SMIN_J_NEXT]] = call i32 @llvm.smin.i32(i32 [[A]], i32 [[SMIN_J]])
 ; CHECK-NEXT:    [[J_INC:%.*]] = add i32 [[J]], 1
 ; CHECK-NEXT:    [[CMP_J:%.*]] = icmp slt i32 [[J_INC]], 2
 ; CHECK-NEXT:    br label %[[FOR_I_LATCH]]
@@ -271,8 +270,7 @@ for.j:
   %smin.j = phi i32 [ %smin.i, %for.i.header ], [ %smin.j.next, %for.j ]
   %idx = getelementptr inbounds [2 x [2 x i32]], ptr %A, i32 0, i32 %j, i32 %i
   %a = load i32, ptr %idx, align 4
-  %cmp = icmp slt i32 %a, %smin.j
-  %smin.j.next = select i1 %cmp, i32 %a, i32 %smin.j
+  %smin.j.next = call i32 @llvm.smin(i32 %a, i32 %smin.j)
   %j.inc = add i32 %j, 1
   %cmp.j = icmp slt i32 %j.inc, 2
   br i1 %cmp.j, label %for.j, label %for.i.latch
@@ -314,8 +312,7 @@ define void @reduction_smax(ptr %A, i32 %init) {
 ; CHECK:       [[FOR_J_SPLIT1]]:
 ; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds [2 x [2 x i32]], ptr [[A]], i32 0, i32 [[J]], i32 [[I]]
 ; CHECK-NEXT:    [[A:%.*]] = load i32, ptr [[IDX]], align 4
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A]], [[SMAX_J]]
-; CHECK-NEXT:    [[SMAX_J_NEXT]] = select i1 [[CMP]], i32 [[A]], i32 [[SMAX_J]]
+; CHECK-NEXT:    [[SMAX_J_NEXT]] = call i32 @llvm.smax.i32(i32 [[A]], i32 [[SMAX_J]])
 ; CHECK-NEXT:    [[J_INC:%.*]] = add i32 [[J]], 1
 ; CHECK-NEXT:    [[CMP_J:%.*]] = icmp slt i32 [[J_INC]], 2
 ; CHECK-NEXT:    br label %[[FOR_I_LATCH]]
@@ -344,8 +341,7 @@ for.j:
   %smax.j = phi i32 [ %smax.i, %for.i.header ], [ %smax.j.next, %for.j ]
   %idx = getelementptr inbounds [2 x [2 x i32]], ptr %A, i32 0, i32 %j, i32 %i
   %a = load i32, ptr %idx, align 4
-  %cmp = icmp sgt i32 %a, %smax.j
-  %smax.j.next = select i1 %cmp, i32 %a, i32 %smax.j
+  %smax.j.next = call i32 @llvm.smax(i32 %a, i32 %smax.j)
   %j.inc = add i32 %j, 1
   %cmp.j = icmp slt i32 %j.inc, 2
   br i1 %cmp.j, label %for.j, label %for.i.latch
@@ -387,8 +383,7 @@ define void @reduction_umin(ptr %A, i32 %init) {
 ; CHECK:       [[FOR_J_SPLIT1]]:
 ; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds [2 x [2 x i32]], ptr [[A]], i32 0, i32 [[J]], i32 [[I]]
 ; CHECK-NEXT:    [[A:%.*]] = load i32, ptr [[IDX]], align 4
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A]], [[UMIN_J]]
-; CHECK-NEXT:    [[UMIN_J_NEXT]] = select i1 [[CMP]], i32 [[A]], i32 [[UMIN_J]]
+; CHECK-NEXT:    [[UMIN_J_NEXT]] = call i32 @llvm.umin.i32(i32 [[A]], i32 [[UMIN_J]])
 ; CHECK-NEXT:    [[J_INC:%.*]] = add i32 [[J]], 1
 ; CHECK-NEXT:    [[CMP_J:%.*]] = icmp slt i32 [[J_INC]], 2
 ; CHECK-NEXT:    br label %[[FOR_I_LATCH]]
@@ -417,8 +412,7 @@ for.j:
   %umin.j = phi i32 [ %umin.i, %for.i.header ], [ %umin.j.next, %for.j ]
   %idx = getelementptr inbounds [2 x [2 x i32]], ptr %A, i32 0, i32 %j, i32 %i
   %a = load i32, ptr %idx, align 4
-  %cmp = icmp ult i32 %a, %umin.j
-  %umin.j.next = select i1 %cmp, i32 %a, i32 %umin.j
+  %umin.j.next = call i32 @llvm.umin(i32 %a, i32 %umin.j)
   %j.inc = add i32 %j, 1
   %cmp.j = icmp slt i32 %j.inc, 2
   br i1 %cmp.j, label %for.j, label %for.i.latch
@@ -460,8 +454,7 @@ define void @reduction_umax(ptr %A) {
 ; CHECK:       [[FOR_J_SPLIT1]]:
 ; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds [2 x [2 x i32]], ptr [[A]], i32 0, i32 [[J]], i32 [[I]]
 ; CHECK-NEXT:    [[A:%.*]] = load i32, ptr [[IDX]], align 4
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[A]], [[UMAX_J]]
-; CHECK-NEXT:    [[UMAX_J_NEXT]] = select i1 [[CMP]], i32 [[A]], i32 [[UMAX_J]]
+; CHECK-NEXT:    [[UMAX_J_NEXT]] = call i32 @llvm.umax.i32(i32 [[A]], i32 [[UMAX_J]])
 ; CHECK-NEXT:    [[J_INC:%.*]] = add i32 [[J]], 1
 ; CHECK-NEXT:    [[CMP_J:%.*]] = icmp slt i32 [[J_INC]], 2
 ; CHECK-NEXT:    br label %[[FOR_I_LATCH]]
@@ -490,8 +483,7 @@ for.j:
   %umax.j = phi i32 [ %umax.i, %for.i.header ], [ %umax.j.next, %for.j ]
   %idx = getelementptr inbounds [2 x [2 x i32]], ptr %A, i32 0, i32 %j, i32 %i
   %a = load i32, ptr %idx, align 4
-  %cmp = icmp ugt i32 %a, %umax.j
-  %umax.j.next = select i1 %cmp, i32 %a, i32 %umax.j
+  %umax.j.next = call i32 @llvm.umax(i32 %a, i32 %umax.j)
   %j.inc = add i32 %j, 1
   %cmp.j = icmp slt i32 %j.inc, 2
   br i1 %cmp.j, label %for.j, label %for.i.latch
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions-min-max.ll b/llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions-min-max.ll
index 4b639b76ea4f8..e0f9131bb64c4 100644
--- a/llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions-min-max.ll
+++ b/llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions-min-max.ll
@@ -14,18 +14,15 @@ define i32 @test_smin_reduction(ptr %a, i64 %n) {
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MIN_1:%.*]] = phi i32 [ 2147483647, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MIN:%.*]] = phi i32 [ 2147483647, %[[ENTRY_NEW]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN:%.*]] = phi i32 [ 2147483647, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[GEP_A]], align 4
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[MIN]], [[TMP2]]
-; CHECK-NEXT:    [[RDX_NEXT]] = select i1 [[CMP]], i32 [[MIN]], i32 [[TMP2]]
+; CHECK-NEXT:    [[RDX_NEXT:%.*]] = call i32 @llvm.smin.i32(i32 [[MIN]], i32 [[TMP2]])
 ; CHECK-NEXT:    [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[GEP_A_1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV_NEXT]]
+; CHECK-NEXT:    [[GEP_A_1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[IV_NEXT]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[GEP_A_1]], align 4
-; CHECK-NEXT:    [[CMP_1:%.*]] = icmp slt i32 [[MIN_1]], [[TMP3]]
-; CHECK-NEXT:    [[RDX_NEXT_1]] = select i1 [[CMP_1]], i32 [[MIN_1]], i32 [[TMP3]]
+; CHECK-NEXT:    [[RDX_NEXT_1]] = call i32 @llvm.smin.i32(i32 [[RDX_NEXT]], i32 [[TMP3]])
 ; CHECK-NEXT:    [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
 ; CHECK-NEXT:    [[NITER_NEXT_1]] = add i64 [[NITER]], 2
 ; CHECK-NEXT:    [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
@@ -34,23 +31,21 @@ define i32 @test_smin_reduction(ptr %a, i64 %n) {
 ; CHECK-NEXT:    [[RES_PH:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[MIN_UNR:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]
-; CHECK-NEXT:    [[RDX_MINMAX:%.*]] = call i32 @llvm.smin.i32(i32 [[RDX_NEXT]], i32 [[RDX_NEXT_1]])
 ; CHECK-NEXT:    [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
 ; CHECK-NEXT:    br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
 ; CHECK:       [[LOOP_EPIL_PREHEADER]]:
 ; CHECK-NEXT:    [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
-; CHECK-NEXT:    [[MIN_EPIL_INIT:%.*]] = phi i32 [ 2147483647, %[[ENTRY]] ], [ [[RDX_MINMAX]], %[[EXIT_UNR_LCSSA]] ]
+; CHECK-NEXT:    [[MIN_EPIL_INIT:%.*]] = phi i32 [ 2147483647, %[[ENTRY]] ], [ [[MIN_UNR]], %[[EXIT_UNR_LCSSA]] ]
 ; CHECK-NEXT:    [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[LCMP_MOD2]])
 ; CHECK-NEXT:    br label %[[LOOP_EPIL:.*]]
 ; CHECK:       [[LOOP_EPIL]]:
-; CHECK-NEXT:    [[GEP_A_EPIL:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV_EPIL_INIT]]
+; CHECK-NEXT:    [[GEP_A_EPIL:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[IV_EPIL_INIT]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[GEP_A_EPIL]], align 4
-; CHECK-NEXT:    [[CMP_EPIL:%.*]] = icmp slt i32 [[MIN_EPIL_INIT]], [[TMP4]]
-; CHECK-NEXT:    [[RDX_NEXT_EPIL:%.*]] = select i1 [[CMP_EPIL]], i32 [[MIN_EPIL_INIT]], i32 [[TMP4]]
+; CHECK-NEXT:    [[RDX_NEXT_EPIL:%.*]] = call i32 @llvm.smin.i32(i32 [[MIN_EPIL_INIT]], i32 [[TMP4]])
 ; CHECK-NEXT:    br label %[[EXIT]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[RDX_MINMAX]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[RES_PH]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
 entry:
@@ -59,10 +54,9 @@ entry:
 loop:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
   %min = phi i32 [ 2147483647, %entry ], [ %rdx.next, %loop ]
-  %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
-  %1 = load i32, ptr %gep.a, align 4
-  %cmp = icmp slt i32 %min, %1
-  %rdx.next = select i1 %cmp, i32 %min, i32 %1
+  %gep.a = getelementptr inbounds nuw [4 x i8], ptr %a, i64 %iv
+  %0 = load i32, ptr %gep.a, align 4
+  %rdx.next = call i32 @llvm.smin.i32(i32 %min, i32 %0)
   %iv.next = add nuw nsw i64 %iv, 1
   %ec = icmp eq i64 %iv.next, %n
   br i1 %ec, label %exit, label %loop, !llvm.loop !0
@@ -85,18 +79,15 @@ define i32 @test_smax_reduction(ptr %a, i64 %n) {
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX_1:%.*]] = phi i32 [ -2147483648, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi i32 [ -2147483648, %[[ENTRY_NEW]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi i32 [ -2147483648, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[GEP_A]], align 4
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[MAX]], [[TMP2]]
-; CHECK-NEXT:    [[RDX_NEXT]] = select i1 [[CMP]], i32 [[MAX]], i32 [[TMP2]]
+; CHECK-NEXT:    [[RDX_NEXT:%.*]] = call i32 @llvm.smax.i32(i32 [[MAX]], i32 [[TMP2]])
 ; CHECK-NEXT:    [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[GEP_A_1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV_NEXT]]
+; CHECK-NEXT:    [[GEP_A_1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[IV_NEXT]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[GEP_A_1]], align 4
-; CHECK-NEXT:    [[CMP_1:%.*]] = icmp sgt i32 [[MAX_1]], [[TMP3]]
-; CHECK-NEXT:    [[RDX_NEXT_1]] = select i1 [[CMP_1]], i32 [[MAX_1]], i32 [[TMP3]]
+; CHECK-NEXT:    [[RDX_NEXT_1]] = call i32 @llvm.smax.i32(i32 [[RDX_NEXT]], i32 [[TMP3]])
 ; CHECK-NEXT:    [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
 ; CHECK-NEXT:    [[NITER_NEXT_1]] = add i64 [[NITER]], 2
 ; CHECK-NEXT:    [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
@@ -105,23 +96,21 @@ define i32 @test_smax_reduction(ptr %a, i64 %n) {
 ; CHECK-NEXT:    [[RES_PH:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[MAX_UNR:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]
-; CHECK-NEXT:    [[RDX_MINMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[RDX_NEXT]], i32 [[RDX_NEXT_1]])
 ; CHECK-NEXT:    [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
 ; CHECK-NEXT:    br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
 ; CHECK:       [[LOOP_EPIL_PREHEADER]]:
 ; CHECK-NEXT:    [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
-; CHECK-NEXT:    [[MAX_EPIL_INIT:%.*]] = phi i32 [ -2147483648, %[[ENTRY]] ], [ [[RDX_MINMAX]], %[[EXIT_UNR_LCSSA]] ]
+; CHECK-NEXT:    [[MAX_EPIL_INIT:%.*]] = phi i32 [ -2147483648, %[[ENTRY]] ], [ [[MAX_UNR]], %[[EXIT_UNR_LCSSA]] ]
 ; CHECK-NEXT:    [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[LCMP_MOD2]])
 ; CHECK-NEXT:    br label %[[LOOP_EPIL:.*]]
 ; CHECK:       [[LOOP_EPIL]]:
-; CHECK-NEXT:    [[GEP_A_EPIL:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV_EPIL_INIT]]
+; CHECK-NEXT:    [[GEP_A_EPIL:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[IV_EPIL_INIT]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[GEP_A_EPIL]], align 4
-; CHECK-NEXT:    [[CMP_EPIL:%.*]] = icmp sgt i32 [[MAX_EPIL_INIT]], [[TMP4]]
-; CHECK-NEXT:    [[RDX_NEXT_EPIL:%.*]] = select i1 [[CMP_EPIL]], i32 [[MAX_EPIL_INIT]], i32 [[TMP4]]
+; CHECK-NEXT:    [[RDX_NEXT_EPIL:%.*]] = call i32 @llvm.smax.i32(i32 [[MAX_EPIL_INIT]], i32 [[TMP4]])
 ; CHECK-NEXT:    br label %[[EXIT]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[RDX_MINMAX]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[RES_PH]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
 entry:
@@ -130,10 +119,9 @@ entry:
 loop:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
   %max = phi i32 [ -2147483648, %entry ], [ %rdx.next, %loop ]
-  %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
-  %1 = load i32, ptr %gep.a, align 4
-  %cmp = icmp sgt i32 %max, %1
-  %rdx.next = select i1 %cmp, i32 %max, i32 %1
+  %gep.a = getelementptr inbounds nuw [4 x i8], ptr %a, i64 %iv
+  %0 = load i32, ptr %gep.a, align 4
+  %rdx.next = call i32 @llvm.smax.i32(i32 %max, i32 %0)
   %iv.next = add nuw nsw i64 %iv, 1
   %ec = icmp eq i64 %iv.next, %n
   br i1 %ec, label %exit, label %loop, !llvm.loop !0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr33053.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr33053.ll
index 8a0cfa74a3ee6..a72fdeb6f10fe 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/pr33053.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr33053.ll
@@ -23,10 +23,8 @@ for.body:
   %c.042 = phi i16 [ 0, %for.body.lr.ph ], [ %c.0., %for.body ]
   %arrayidx = getelementptr inbounds i16, ptr %1, i64 %indvars.iv
   %4 = load i16, ptr %arrayidx, align 2, !tbaa !7
-  %cmp2 = icmp sgt i16 %c.042, %4
-  %c.0. = select i1 %cmp2, i16 %c.042, i16 %4
-  %cmp13 = icmp slt i16 %d.043, %4
-  %.sink28 = select i1 %cmp13, i16 %d.043, i16 %4
+  %c.0. = call i16 @llvm.smax(i16 %c.042, i16 %4)
+  %.sink28 = call i16 @llvm.smin(i16 %d.043, i16 %4)
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %cmp = icmp slt i64 %indvars.iv.next, %3
   br i1 %cmp, label %for.body, label %for.end
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reductions.ll
index 6adb5e5dd8a1f..5f43b71e1444c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reductions.ll
@@ -132,10 +132,8 @@ define i32 @smin(ptr nocapture %a, ptr nocapture readonly %b, i64 %n) {
 ; CHECK: vector.body:
 ; CHECK: %[[LOAD1:.*]] = load <vscale x 8 x i32>
 ; CHECK: %[[LOAD2:.*]] = load <vscale x 8 x i32>
-; CHECK: %[[ICMP1:.*]] = icmp slt <vscale x 8 x i32> %[[LOAD1]]
-; CHECK: %[[ICMP2:.*]] = icmp slt <vscale x 8 x i32> %[[LOAD2]]
-; CHECK: %[[SEL1:.*]] = select <vscale x 8 x i1> %[[ICMP1]], <vscale x 8 x i32> %[[LOAD1]]
-; CHECK: %[[SEL2:.*]] = select <vscale x 8 x i1> %[[ICMP2]], <vscale x 8 x i32> %[[LOAD2]]
+; CHECK: %[[SEL1:.*]] = call <vscale x 8 x i32> @llvm.smin
+; CHECK: %[[SEL2:.*]] = call <vscale x 8 x i32> @llvm.smin
 ; CHECK: middle.block:
 ; CHECK: %[[RDX:.*]] = call <vscale x 8 x i32> @llvm.smin.nxv8i32(<vscale x 8 x i32> %[[SEL1]], <vscale x 8 x i32> %[[SEL2]])
 ; CHECK-NEXT: call i32 @llvm.vector.reduce.smin.nxv8i32(<vscale x 8 x i32>  %[[RDX]])
@@ -147,8 +145,7 @@ for.body:
   %sum.010 = phi i32 [ 2, %entry ], [ %.sroa.speculated, %for.body ]
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv
   %0 = load i32, ptr %arrayidx, align 4
-  %cmp.i = icmp slt i32 %0, %sum.010
-  %.sroa.speculated = select i1 %cmp.i, i32 %0, i32 %sum.010
+  %.sroa.speculated = call i32 @llvm.smin(i32 %0, i32 %sum.010)
   %iv.next = add nuw nsw i64 %iv, 1
   %exitcond.not = icmp eq i64 %iv.next, %n
   br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0
@@ -165,10 +162,8 @@ define i32 @umax(ptr nocapture %a, ptr nocapture readonly %b, i64 %n) {
 ; CHECK: vector.body:
 ; CHECK: %[[LOAD1:.*]] = load <vscale x 8 x i32>
 ; CHECK: %[[LOAD2:.*]] = load <vscale x 8 x i32>
-; CHECK: %[[ICMP1:.*]] = icmp ugt <vscale x 8 x i32> %[[LOAD1]]
-; CHECK: %[[ICMP2:.*]] = icmp ugt <vscale x 8 x i32> %[[LOAD2]]
-; CHECK: %[[SEL1:.*]] = select <vscale x 8 x i1> %[[ICMP1]], <vscale x 8 x i32> %[[LOAD1]]
-; CHECK: %[[SEL2:.*]] = select <vscale x 8 x i1> %[[ICMP2]], <vscale x 8 x i32> %[[LOAD2]]
+; CHECK: %[[SEL1:.*]] = call <vscale x 8 x i32> @llvm.umax
+; CHECK: %[[SEL2:.*]] = call <vscale x 8 x i32> @llvm.umax
 ; CHECK: middle.block:
 ; CHECK: %[[RDX:.*]] = call <vscale x 8 x i32> @llvm.umax.nxv8i32(<vscale x 8 x i32> %[[SEL1]], <vscale x 8 x i32> %[[SEL2]])
 ; CHECK-NEXT: call i32 @llvm.vector.reduce.umax.nxv8i32(<vscale x 8 x i32>  %[[RDX]])
@@ -180,8 +175,7 @@ for.body:
   %sum.010 = phi i32 [ 2, %entry ], [ %.sroa.speculated, %for.body ]
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv
   %0 = load i32, ptr %arrayidx, align 4
-  %cmp.i = icmp ugt i32 %0, %sum.010
-  %.sroa.speculated = select i1 %cmp.i, i32 %0, i32 %sum.010
+  %.sroa.speculated = call i32 @llvm.umax(i32 %0, i32 %sum.010)
   %iv.next = add nuw nsw i64 %iv, 1
   %exitcond.not = icmp eq i64 %iv.next, %n
   br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-predselect.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-predselect.ll
index a56524697a540..e564151015e63 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-predselect.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-predselect.ll
@@ -393,8 +393,7 @@ define i32 @reduction_min(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 1000), [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT:    [[TMP1]] = select <4 x i1> [[TMP3]], <4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP1]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
@@ -408,8 +407,7 @@ define i32 @reduction_min(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[RESULT_08:%.*]] = phi i32 [ [[V0:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[L0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[C0:%.*]] = icmp slt i32 [[RESULT_08]], [[L0]]
-; CHECK-NEXT:    [[V0]] = select i1 [[C0]], i32 [[RESULT_08]], i32 [[L0]]
+; CHECK-NEXT:    [[V0]] = call i32 @llvm.smin.i32(i32 [[RESULT_08]], i32 [[L0]])
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i32 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], 257
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
@@ -425,8 +423,7 @@ for.body:
   %result.08 = phi i32 [ %v0, %for.body ], [ 1000, %entry ]
   %arrayidx = getelementptr inbounds i32, ptr %A, i32 %indvars.iv
   %l0 = load i32, ptr %arrayidx, align 4
-  %c0 = icmp slt i32 %result.08, %l0
-  %v0 = select i1 %c0, i32 %result.08, i32 %l0
+  %v0 = call i32 @llvm.smin(i32 %result.08, i32 %l0)
   %indvars.iv.next = add i32 %indvars.iv, 1
   %exitcond = icmp eq i32 %indvars.iv.next, 257
   br i1 %exitcond, label %for.end, label %for.body
@@ -447,8 +444,7 @@ define i32 @reduction_max(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 1000), [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT:    [[TMP1]] = select <4 x i1> [[TMP3]], <4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP1]] = call <4 x i32> @llvm.umax.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
@@ -462,8 +458,7 @@ define i32 @reduction_max(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[RESULT_08:%.*]] = phi i32 [ [[V0:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[L0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[C0:%.*]] = icmp ugt i32 [[RESULT_08]], [[L0]]
-; CHECK-NEXT:    [[V0]] = select i1 [[C0]], i32 [[RESULT_08]], i32 [[L0]]
+; CHECK-NEXT:    [[V0]] = call i32 @llvm.umax.i32(i32 [[RESULT_08]], i32 [[L0]])
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i32 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], 257
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
@@ -479,8 +474,7 @@ for.body:
   %result.08 = phi i32 [ %v0, %for.body ], [ 1000, %entry ]
   %arrayidx = getelementptr inbounds i32, ptr %A, i32 %indvars.iv
   %l0 = load i32, ptr %arrayidx, align 4
-  %c0 = icmp ugt i32 %result.08, %l0
-  %v0 = select i1 %c0, i32 %result.08, i32 %l0
+  %v0 = call i32 @llvm.umax(i32 %result.08, i32 %l0)
   %indvars.iv.next = add i32 %indvars.iv, 1
   %exitcond = icmp eq i32 %indvars.iv.next, 257
   br i1 %exitcond, label %for.end, label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-types.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-types.ll
index a314e4a2e82c2..4f91e3331bab1 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-types.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-types.ll
@@ -500,8 +500,7 @@ define i32 @smin_i32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 2147483647), [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT:    [[TMP4]] = select <4 x i1> [[TMP3]], <4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
@@ -518,8 +517,7 @@ define i32 @smin_i32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[I_08]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[R_07]], [[TMP7]]
-; CHECK-NEXT:    [[ADD]] = select i1 [[C]], i32 [[R_07]], i32 [[TMP7]]
+; CHECK-NEXT:    [[ADD]] = call i32 @llvm.smin.i32(i32 [[R_07]], i32 [[TMP7]])
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
@@ -539,8 +537,7 @@ for.body:
   %r.07 = phi i32 [ %add, %for.body ], [ 2147483647, %entry ]
   %arrayidx = getelementptr inbounds i32, ptr %x, i32 %i.08
   %0 = load i32, ptr %arrayidx, align 4
-  %c = icmp slt i32 %r.07, %0
-  %add = select i1 %c, i32 %r.07, i32 %0
+  %add = call i32 @llvm.smin(i32 %r.07, i32 %0)
   %inc = add nuw nsw i32 %i.08, 1
   %exitcond = icmp eq i32 %inc, %n
   br i1 %exitcond, label %for.cond.cleanup, label %for.body
@@ -567,8 +564,7 @@ define i32 @smax_i32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -2147483648), [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT:    [[TMP4]] = select <4 x i1> [[TMP3]], <4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
@@ -585,8 +581,7 @@ define i32 @smax_i32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[I_08]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[R_07]], [[TMP7]]
-; CHECK-NEXT:    [[ADD]] = select i1 [[C]], i32 [[R_07]], i32 [[TMP7]]
+; CHECK-NEXT:    [[ADD]] = call i32 @llvm.smax.i32(i32 [[R_07]], i32 [[TMP7]])
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
@@ -606,8 +601,7 @@ for.body:
   %r.07 = phi i32 [ %add, %for.body ], [ -2147483648, %entry ]
   %arrayidx = getelementptr inbounds i32, ptr %x, i32 %i.08
   %0 = load i32, ptr %arrayidx, align 4
-  %c = icmp sgt i32 %r.07, %0
-  %add = select i1 %c, i32 %r.07, i32 %0
+  %add = call i32 @llvm.smax(i32 %r.07, i32 %0)
   %inc = add nuw nsw i32 %i.08, 1
   %exitcond = icmp eq i32 %inc, %n
   br i1 %exitcond, label %for.cond.cleanup, label %for.body
@@ -634,8 +628,7 @@ define i32 @umin_i32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT:    [[TMP4]] = select <4 x i1> [[TMP3]], <4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4]] = call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
@@ -652,8 +645,7 @@ define i32 @umin_i32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[I_08]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[R_07]], [[TMP7]]
-; CHECK-NEXT:    [[ADD]] = select i1 [[C]], i32 [[R_07]], i32 [[TMP7]]
+; CHECK-NEXT:    [[ADD]] = call i32 @llvm.umin.i32(i32 [[R_07]], i32 [[TMP7]])
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
@@ -673,8 +665,7 @@ for.body:
   %r.07 = phi i32 [ %add, %for.body ], [ 4294967295, %entry ]
   %arrayidx = getelementptr inbounds i32, ptr %x, i32 %i.08
   %0 = load i32, ptr %arrayidx, align 4
-  %c = icmp ult i32 %r.07, %0
-  %add = select i1 %c, i32 %r.07, i32 %0
+  %add = call i32 @llvm.umin(i32 %r.07, i32 %0)
   %inc = add nuw nsw i32 %i.08, 1
   %exitcond = icmp eq i32 %inc, %n
   br i1 %exitcond, label %for.cond.cleanup, label %for.body
@@ -701,8 +692,7 @@ define i32 @umax_i32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT:    [[TMP4]] = select <4 x i1> [[TMP3]], <4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4]] = call <4 x i32> @llvm.umax.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
@@ -719,8 +709,7 @@ define i32 @umax_i32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[I_08]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[R_07]], [[TMP7]]
-; CHECK-NEXT:    [[ADD]] = select i1 [[C]], i32 [[R_07]], i32 [[TMP7]]
+; CHECK-NEXT:    [[ADD]] = call i32 @llvm.umax.i32(i32 [[R_07]], i32 [[TMP7]])
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
@@ -740,8 +729,7 @@ for.body:
   %r.07 = phi i32 [ %add, %for.body ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds i32, ptr %x, i32 %i.08
   %0 = load i32, ptr %arrayidx, align 4
-  %c = icmp ugt i32 %r.07, %0
-  %add = select i1 %c, i32 %r.07, i32 %0
+  %add = call i32 @llvm.umax(i32 %r.07, i32 %0)
   %inc = add nuw nsw i32 %i.08, 1
   %exitcond = icmp eq i32 %inc, %n
   br i1 %exitcond, label %for.cond.cleanup, label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/tail-fold-multiple-icmps.ll b/llvm/test/Transforms/LoopVectorize/ARM/tail-fold-multiple-icmps.ll
index 6185c152ddc22..1a8f3dd4f1e84 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/tail-fold-multiple-icmps.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/tail-fold-multiple-icmps.ll
@@ -21,10 +21,8 @@ define arm_aapcs_vfpcc i32 @minmaxval4(ptr nocapture readonly %x, ptr nocapture
 ; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 -2147483648), [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], [[VEC_PHI1]]
-; CHECK-NEXT:    [[TMP1]] = select <4 x i1> [[TMP7]], <4 x i32> [[WIDE_LOAD]], <4 x i32> [[VEC_PHI1]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT:    [[TMP2]] = select <4 x i1> [[TMP8]], <4 x i32> [[WIDE_LOAD]], <4 x i32> [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP1]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[WIDE_LOAD]], <4 x i32> [[VEC_PHI1]])
+; CHECK-NEXT:    [[TMP2]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[WIDE_LOAD]], <4 x i32> [[VEC_PHI]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -53,10 +51,8 @@ define arm_aapcs_vfpcc i32 @minmaxval4(ptr nocapture readonly %x, ptr nocapture
 ; CHECK-NEXT:    [[MAX_027:%.*]] = phi i32 [ [[COND]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX2]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[I_029]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP6]], [[MAX_027]]
-; CHECK-NEXT:    [[COND]] = select i1 [[CMP1]], i32 [[TMP6]], i32 [[MAX_027]]
-; CHECK-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP6]], [[MIN_028]]
-; CHECK-NEXT:    [[COND9]] = select i1 [[CMP4]], i32 [[TMP6]], i32 [[MIN_028]]
+; CHECK-NEXT:    [[COND]] = call i32 @llvm.smax.i32(i32 [[TMP6]], i32 [[MAX_027]])
+; CHECK-NEXT:    [[COND9]] = call i32 @llvm.smin.i32(i32 [[TMP6]], i32 [[MIN_028]])
 ; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_029]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
@@ -77,10 +73,8 @@ for.body:
   %max.027 = phi i32 [ %cond, %for.body ], [ -2147483648, %entry ]
   %arrayidx = getelementptr inbounds i32, ptr %x, i32 %i.029
   %0 = load i32, ptr %arrayidx, align 4
-  %cmp1 = icmp sgt i32 %0, %max.027
-  %cond = select i1 %cmp1, i32 %0, i32 %max.027
-  %cmp4 = icmp slt i32 %0, %min.028
-  %cond9 = select i1 %cmp4, i32 %0, i32 %min.028
+  %cond = call i32 @llvm.smax(i32 %0, i32 %max.027)
+  %cond9 = call i32 @llvm.smin(i32 %0, i32 %min.028)
   %inc = add nuw i32 %i.029, 1
   %exitcond.not = icmp eq i32 %inc, %N
   br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-not-allowed.ll b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-not-allowed.ll
index 384ecd5e11782..36058a5004003 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-not-allowed.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-not-allowed.ll
@@ -579,8 +579,7 @@ define i32 @i32_smin_reduction(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 2147483647), [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT:    [[TMP4]] = select <4 x i1> [[TMP3]], <4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
@@ -597,8 +596,7 @@ define i32 @i32_smin_reduction(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[I_08]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[R_07]], [[TMP7]]
-; CHECK-NEXT:    [[ADD]] = select i1 [[C]], i32 [[R_07]], i32 [[TMP7]]
+; CHECK-NEXT:    [[ADD]] = call i32 @llvm.smin.i32(i32 [[R_07]], i32 [[TMP7]])
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
@@ -618,8 +616,7 @@ for.body:
   %r.07 = phi i32 [ %add, %for.body ], [ 2147483647, %entry ]
   %arrayidx = getelementptr inbounds i32, ptr %x, i32 %i.08
   %0 = load i32, ptr %arrayidx, align 4
-  %c = icmp slt i32 %r.07, %0
-  %add = select i1 %c, i32 %r.07, i32 %0
+  %add = call i32 @llvm.smin(i32 %r.07, i32 %0)
   %inc = add nuw nsw i32 %i.08, 1
   %exitcond = icmp eq i32 %inc, %n
   br i1 %exitcond, label %for.cond.cleanup, label %for.body
@@ -646,8 +643,7 @@ define i32 @i32_smax_reduction(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -2147483648), [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT:    [[TMP4]] = select <4 x i1> [[TMP3]], <4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
@@ -664,8 +660,7 @@ define i32 @i32_smax_reduction(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[I_08]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[R_07]], [[TMP7]]
-; CHECK-NEXT:    [[ADD]] = select i1 [[C]], i32 [[R_07]], i32 [[TMP7]]
+; CHECK-NEXT:    [[ADD]] = call i32 @llvm.smax.i32(i32 [[R_07]], i32 [[TMP7]])
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
@@ -685,8 +680,7 @@ for.body:
   %r.07 = phi i32 [ %add, %for.body ], [ -2147483648, %entry ]
   %arrayidx = getelementptr inbounds i32, ptr %x, i32 %i.08
   %0 = load i32, ptr %arrayidx, align 4
-  %c = icmp sgt i32 %r.07, %0
-  %add = select i1 %c, i32 %r.07, i32 %0
+  %add = call i32 @llvm.smax(i32 %r.07, i32 %0)
   %inc = add nuw nsw i32 %i.08, 1
   %exitcond = icmp eq i32 %inc, %n
   br i1 %exitcond, label %for.cond.cleanup, label %for.body
@@ -713,8 +707,7 @@ define i32 @i32_umin_reduction(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT:    [[TMP4]] = select <4 x i1> [[TMP3]], <4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4]] = call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
@@ -731,8 +724,7 @@ define i32 @i32_umin_reduction(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[I_08]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[R_07]], [[TMP7]]
-; CHECK-NEXT:    [[ADD]] = select i1 [[C]], i32 [[R_07]], i32 [[TMP7]]
+; CHECK-NEXT:    [[ADD]] = call i32 @llvm.umin.i32(i32 [[R_07]], i32 [[TMP7]])
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
@@ -752,8 +744,7 @@ for.body:
   %r.07 = phi i32 [ %add, %for.body ], [ 4294967295, %entry ]
   %arrayidx = getelementptr inbounds i32, ptr %x, i32 %i.08
   %0 = load i32, ptr %arrayidx, align 4
-  %c = icmp ult i32 %r.07, %0
-  %add = select i1 %c, i32 %r.07, i32 %0
+  %add = call i32 @llvm.umin(i32 %r.07, i32 %0)
   %inc = add nuw nsw i32 %i.08, 1
   %exitcond = icmp eq i32 %inc, %n
   br i1 %exitcond, label %for.cond.cleanup, label %for.body
@@ -780,8 +771,7 @@ define i32 @i32_umax_reduction(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT:    [[TMP4]] = select <4 x i1> [[TMP3]], <4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4]] = call <4 x i32> @llvm.umax.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
@@ -798,8 +788,7 @@ define i32 @i32_umax_reduction(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[I_08]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[R_07]], [[TMP7]]
-; CHECK-NEXT:    [[ADD]] = select i1 [[C]], i32 [[R_07]], i32 [[TMP7]]
+; CHECK-NEXT:    [[ADD]] = call i32 @llvm.umax.i32(i32 [[R_07]], i32 [[TMP7]])
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
@@ -819,8 +808,7 @@ for.body:
   %r.07 = phi i32 [ %add, %for.body ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds i32, ptr %x, i32 %i.08
   %0 = load i32, ptr %arrayidx, align 4
-  %c = icmp ugt i32 %r.07, %0
-  %add = select i1 %c, i32 %r.07, i32 %0
+  %add = call i32 @llvm.umax(i32 %r.07, i32 %0)
   %inc = add nuw nsw i32 %i.08, 1
   %exitcond = icmp eq i32 %inc, %n
   br i1 %exitcond, label %for.cond.cleanup, label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
index 88be959a70b3a..022f491eb8d04 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
@@ -213,8 +213,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; OUTLOOP-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
 ; OUTLOOP-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; OUTLOOP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
-; OUTLOOP-NEXT:    [[TMP9:%.*]] = icmp slt <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; OUTLOOP-NEXT:    [[TMP10]] = select <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
+; OUTLOOP-NEXT:    [[TMP10]] = call <vscale x 4 x i32> @llvm.smin.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
 ; OUTLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; OUTLOOP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; OUTLOOP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -231,8 +230,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; OUTLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SMIN:%.*]], [[FOR_BODY]] ]
 ; OUTLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; OUTLOOP-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; OUTLOOP-NEXT:    [[CMP_I:%.*]] = icmp slt i32 [[TMP13]], [[RDX]]
-; OUTLOOP-NEXT:    [[SMIN]] = select i1 [[CMP_I]], i32 [[TMP13]], i32 [[RDX]]
+; OUTLOOP-NEXT:    [[SMIN]] = call i32 @llvm.smin.i32(i32 [[TMP13]], i32 [[RDX]])
 ; OUTLOOP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; OUTLOOP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; OUTLOOP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -252,18 +250,20 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; INLOOP-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
 ; INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; INLOOP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
+; INLOOP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; INLOOP:       vector.body:
 ; INLOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; INLOOP-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ [[START:%.*]], [[VECTOR_PH]] ], [ [[RDX_MINMAX:%.*]], [[VECTOR_BODY]] ]
+; INLOOP-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
 ; INLOOP-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; INLOOP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
-; INLOOP-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
-; INLOOP-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.smin.i32(i32 [[TMP9]], i32 [[VEC_PHI]])
+; INLOOP-NEXT:    [[TMP5]] = call <vscale x 4 x i32> @llvm.smin.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
 ; INLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; INLOOP-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; INLOOP-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; INLOOP:       middle.block:
+; INLOOP-NEXT:    [[RDX_MINMAX:%.*]] = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> [[TMP5]])
 ; INLOOP-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; INLOOP-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; INLOOP:       scalar.ph:
@@ -275,8 +275,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; INLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SMIN:%.*]], [[FOR_BODY]] ]
 ; INLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; INLOOP-NEXT:    [[TMP11:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; INLOOP-NEXT:    [[CMP_I:%.*]] = icmp slt i32 [[TMP11]], [[RDX]]
-; INLOOP-NEXT:    [[SMIN]] = select i1 [[CMP_I]], i32 [[TMP11]], i32 [[RDX]]
+; INLOOP-NEXT:    [[SMIN]] = call i32 @llvm.smin.i32(i32 [[TMP11]], i32 [[RDX]])
 ; INLOOP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; INLOOP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; INLOOP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -298,8 +297,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-OUTLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
-; IF-EVL-OUTLOOP-NEXT:    [[TMP13:%.*]] = icmp slt <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_PHI]]
-; IF-EVL-OUTLOOP-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.smin.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP15]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP14]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-OUTLOOP-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
@@ -316,22 +314,25 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-NEXT:  entry:
 ; IF-EVL-INLOOP-NEXT:    br label [[VECTOR_PH:%.*]]
 ; IF-EVL-INLOOP:       vector.ph:
+; IF-EVL-INLOOP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
+; IF-EVL-INLOOP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL-INLOOP:       vector.body:
 ; IF-EVL-INLOOP-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-INLOOP-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ [[START:%.*]], [[VECTOR_PH]] ], [ [[RDX_MINMAX:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-INLOOP-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-INLOOP-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-INLOOP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-INLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
-; IF-EVL-INLOOP-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vp.reduce.smin.nxv4i32(i32 2147483647, <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
-; IF-EVL-INLOOP-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.smin.i32(i32 [[TMP13]], i32 [[VEC_PHI]])
+; IF-EVL-INLOOP-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.smin.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
+; IF-EVL-INLOOP-NEXT:    [[TMP3]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-INLOOP-NEXT:    [[TMP14:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-INLOOP-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP14]], [[EVL_BASED_IV]]
 ; IF-EVL-INLOOP-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
 ; IF-EVL-INLOOP-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; IF-EVL-INLOOP:       middle.block:
+; IF-EVL-INLOOP-NEXT:    [[RDX_MINMAX:%.*]] = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> [[TMP3]])
 ; IF-EVL-INLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL-INLOOP:       for.end:
 ; IF-EVL-INLOOP-NEXT:    ret i32 [[RDX_MINMAX]]
@@ -403,8 +404,7 @@ for.body:
   %rdx = phi i32 [ %start, %entry ], [ %smin, %for.body ]
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv
   %0 = load i32, ptr %arrayidx, align 4
-  %cmp.i = icmp slt i32 %0, %rdx
-  %smin = select i1 %cmp.i, i32 %0, i32 %rdx
+  %smin = call i32 @llvm.smin(i32 %0, i32 %rdx)
   %iv.next = add nuw nsw i64 %iv, 1
   %exitcond.not = icmp eq i64 %iv.next, %n
   br i1 %exitcond.not, label %for.end, label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reductions.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reductions.ll
index a09ffdeb25691..b8a6f625f6566 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reductions.ll
@@ -298,8 +298,7 @@ define i32 @smin(ptr nocapture %a, ptr nocapture readonly %b, i64 %n) {
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp slt <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT:    [[TMP8:%.*]] = select <vscale x 4 x i1> [[TMP7]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i32> @llvm.smin.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
 ; CHECK-NEXT:    [[TMP9]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP8]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP14]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP14]] to i64
 ; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]
@@ -320,8 +319,7 @@ for.body:
   %sum.010 = phi i32 [ 2, %entry ], [ %.sroa.speculated, %for.body ]
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv
   %0 = load i32, ptr %arrayidx, align 4
-  %cmp.i = icmp slt i32 %0, %sum.010
-  %.sroa.speculated = select i1 %cmp.i, i32 %0, i32 %sum.010
+  %.sroa.speculated = call i32 @llvm.smin(i32 %0, i32 %sum.010)
   %iv.next = add nuw nsw i64 %iv, 1
   %exitcond.not = icmp eq i64 %iv.next, %n
   br i1 %exitcond.not, label %for.end, label %for.body
@@ -346,8 +344,7 @@ define i32 @umax(ptr nocapture %a, ptr nocapture readonly %b, i64 %n) {
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp ugt <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT:    [[TMP8:%.*]] = select <vscale x 4 x i1> [[TMP7]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i32> @llvm.umax.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
 ; CHECK-NEXT:    [[TMP9]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP8]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP14]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP14]] to i64
 ; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]
@@ -368,8 +365,7 @@ for.body:
   %sum.010 = phi i32 [ 2, %entry ], [ %.sroa.speculated, %for.body ]
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv
   %0 = load i32, ptr %arrayidx, align 4
-  %cmp.i = icmp ugt i32 %0, %sum.010
-  %.sroa.speculated = select i1 %cmp.i, i32 %0, i32 %sum.010
+  %.sroa.speculated = call i32 @llvm.umax(i32 %0, i32 %sum.010)
   %iv.next = add nuw nsw i64 %iv, 1
   %exitcond.not = icmp eq i64 %iv.next, %n
   br i1 %exitcond.not, label %for.end, label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-inloop-reduction.ll
index f057a94ed2967..1fad71f9f9e04 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-inloop-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-inloop-reduction.ll
@@ -453,22 +453,25 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ [[START:%.*]], [[VECTOR_PH]] ], [ [[RDX_MINMAX:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
-; IF-EVL-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vp.reduce.smin.nxv4i32(i32 2147483647, <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
-; IF-EVL-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.smin.i32(i32 [[TMP14]], i32 [[VEC_PHI]])
+; IF-EVL-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.smin.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
+; IF-EVL-NEXT:    [[TMP3]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP11]])
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP11]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP8]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
 ; IF-EVL-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; IF-EVL:       middle.block:
+; IF-EVL-NEXT:    [[RDX_MINMAX:%.*]] = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> [[TMP3]])
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    ret i32 [[RDX_MINMAX]]
@@ -483,18 +486,20 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
+; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; NO-VP-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ [[START:%.*]], [[VECTOR_PH]] ], [ [[RDX_MINMAX:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
 ; NO-VP-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
-; NO-VP-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
-; NO-VP-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.smin.i32(i32 [[TMP9]], i32 [[VEC_PHI]])
+; NO-VP-NEXT:    [[TMP4]] = call <vscale x 4 x i32> @llvm.smin.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
 ; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; NO-VP:       middle.block:
+; NO-VP-NEXT:    [[RDX_MINMAX:%.*]] = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> [[TMP4]])
 ; NO-VP-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; NO-VP:       scalar.ph:
@@ -506,8 +511,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SMIN:%.*]], [[FOR_BODY]] ]
 ; NO-VP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; NO-VP-NEXT:    [[CMP_I:%.*]] = icmp slt i32 [[TMP11]], [[RDX]]
-; NO-VP-NEXT:    [[SMIN]] = select i1 [[CMP_I]], i32 [[TMP11]], i32 [[RDX]]
+; NO-VP-NEXT:    [[SMIN]] = call i32 @llvm.smin.i32(i32 [[TMP11]], i32 [[RDX]])
 ; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
@@ -523,8 +527,7 @@ for.body:
   %rdx = phi i32 [ %start, %entry ], [ %smin, %for.body ]
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv
   %0 = load i32, ptr %arrayidx, align 4
-  %cmp.i = icmp slt i32 %0, %rdx
-  %smin = select i1 %cmp.i, i32 %0, i32 %rdx
+  %smin = call i32 @llvm.smin(i32 %0, i32 %rdx)
   %iv.next = add nuw nsw i64 %iv, 1
   %exitcond.not = icmp eq i64 %iv.next, %n
   br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0
@@ -538,22 +541,25 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ [[START:%.*]], [[VECTOR_PH]] ], [ [[RDX_MINMAX:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
-; IF-EVL-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vp.reduce.smax.nxv4i32(i32 -2147483648, <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
-; IF-EVL-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.smax.i32(i32 [[TMP14]], i32 [[VEC_PHI]])
+; IF-EVL-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
+; IF-EVL-NEXT:    [[TMP3]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP11]])
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP11]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP8]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
 ; IF-EVL-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; IF-EVL:       middle.block:
+; IF-EVL-NEXT:    [[RDX_MINMAX:%.*]] = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> [[TMP3]])
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    ret i32 [[RDX_MINMAX]]
@@ -568,18 +574,20 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
+; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; NO-VP-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ [[START:%.*]], [[VECTOR_PH]] ], [ [[RDX_MINMAX:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
 ; NO-VP-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
-; NO-VP-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
-; NO-VP-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.smax.i32(i32 [[TMP9]], i32 [[VEC_PHI]])
+; NO-VP-NEXT:    [[TMP4]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
 ; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; NO-VP:       middle.block:
+; NO-VP-NEXT:    [[RDX_MINMAX:%.*]] = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> [[TMP4]])
 ; NO-VP-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; NO-VP:       scalar.ph:
@@ -591,8 +599,7 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SMAX:%.*]], [[FOR_BODY]] ]
 ; NO-VP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; NO-VP-NEXT:    [[CMP_I:%.*]] = icmp sgt i32 [[TMP11]], [[RDX]]
-; NO-VP-NEXT:    [[SMAX]] = select i1 [[CMP_I]], i32 [[TMP11]], i32 [[RDX]]
+; NO-VP-NEXT:    [[SMAX]] = call i32 @llvm.smax.i32(i32 [[TMP11]], i32 [[RDX]])
 ; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
@@ -608,8 +615,7 @@ for.body:
   %rdx = phi i32 [ %start, %entry ], [ %smax, %for.body ]
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv
   %0 = load i32, ptr %arrayidx, align 4
-  %cmp.i = icmp sgt i32 %0, %rdx
-  %smax = select i1 %cmp.i, i32 %0, i32 %rdx
+  %smax = call i32 @llvm.smax(i32 %0, i32 %rdx)
   %iv.next = add nuw nsw i64 %iv, 1
   %exitcond.not = icmp eq i64 %iv.next, %n
   br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0
@@ -623,22 +629,25 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ [[START:%.*]], [[VECTOR_PH]] ], [ [[RDX_MINMAX:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
-; IF-EVL-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vp.reduce.umin.nxv4i32(i32 -1, <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
-; IF-EVL-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.umin.i32(i32 [[TMP14]], i32 [[VEC_PHI]])
+; IF-EVL-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.umin.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
+; IF-EVL-NEXT:    [[TMP3]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP11]])
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP11]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP8]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
 ; IF-EVL-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; IF-EVL:       middle.block:
+; IF-EVL-NEXT:    [[RDX_MINMAX:%.*]] = call i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32> [[TMP3]])
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    ret i32 [[RDX_MINMAX]]
@@ -653,18 +662,20 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
+; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; NO-VP-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ [[START:%.*]], [[VECTOR_PH]] ], [ [[RDX_MINMAX:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
 ; NO-VP-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
-; NO-VP-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
-; NO-VP-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.umin.i32(i32 [[TMP9]], i32 [[VEC_PHI]])
+; NO-VP-NEXT:    [[TMP4]] = call <vscale x 4 x i32> @llvm.umin.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
 ; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; NO-VP:       middle.block:
+; NO-VP-NEXT:    [[RDX_MINMAX:%.*]] = call i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32> [[TMP4]])
 ; NO-VP-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; NO-VP:       scalar.ph:
@@ -676,8 +687,7 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[UMIN:%.*]], [[FOR_BODY]] ]
 ; NO-VP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; NO-VP-NEXT:    [[CMP_I:%.*]] = icmp ult i32 [[TMP11]], [[RDX]]
-; NO-VP-NEXT:    [[UMIN]] = select i1 [[CMP_I]], i32 [[TMP11]], i32 [[RDX]]
+; NO-VP-NEXT:    [[UMIN]] = call i32 @llvm.umin.i32(i32 [[TMP11]], i32 [[RDX]])
 ; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
@@ -693,8 +703,7 @@ for.body:
   %rdx = phi i32 [ %start, %entry ], [ %umin, %for.body ]
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv
   %0 = load i32, ptr %arrayidx, align 4
-  %cmp.i = icmp ult i32 %0, %rdx
-  %umin = select i1 %cmp.i, i32 %0, i32 %rdx
+  %umin = call i32 @llvm.umin(i32 %0, i32 %rdx)
   %iv.next = add nuw nsw i64 %iv, 1
   %exitcond.not = icmp eq i64 %iv.next, %n
   br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0
@@ -708,22 +717,25 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ [[START:%.*]], [[VECTOR_PH]] ], [ [[RDX_MINMAX:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
-; IF-EVL-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vp.reduce.umax.nxv4i32(i32 0, <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
-; IF-EVL-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.umax.i32(i32 [[TMP14]], i32 [[VEC_PHI]])
+; IF-EVL-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.umax.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
+; IF-EVL-NEXT:    [[TMP3]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP11]])
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP11]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP8]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
 ; IF-EVL-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; IF-EVL:       middle.block:
+; IF-EVL-NEXT:    [[RDX_MINMAX:%.*]] = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> [[TMP3]])
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    ret i32 [[RDX_MINMAX]]
@@ -738,18 +750,20 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
+; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; NO-VP-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ [[START:%.*]], [[VECTOR_PH]] ], [ [[RDX_MINMAX:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
 ; NO-VP-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
-; NO-VP-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
-; NO-VP-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.umax.i32(i32 [[TMP9]], i32 [[VEC_PHI]])
+; NO-VP-NEXT:    [[TMP4]] = call <vscale x 4 x i32> @llvm.umax.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
 ; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; NO-VP:       middle.block:
+; NO-VP-NEXT:    [[RDX_MINMAX:%.*]] = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> [[TMP4]])
 ; NO-VP-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; NO-VP:       scalar.ph:
@@ -761,8 +775,7 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[UMAX:%.*]], [[FOR_BODY]] ]
 ; NO-VP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; NO-VP-NEXT:    [[CMP_I:%.*]] = icmp ugt i32 [[TMP11]], [[RDX]]
-; NO-VP-NEXT:    [[UMAX]] = select i1 [[CMP_I]], i32 [[TMP11]], i32 [[RDX]]
+; NO-VP-NEXT:    [[UMAX]] = call i32 @llvm.umax.i32(i32 [[TMP11]], i32 [[RDX]])
 ; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
@@ -778,8 +791,7 @@ for.body:
   %rdx = phi i32 [ %start, %entry ], [ %umax, %for.body ]
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv
   %0 = load i32, ptr %arrayidx, align 4
-  %cmp.i = icmp ugt i32 %0, %rdx
-  %umax = select i1 %cmp.i, i32 %0, i32 %rdx
+  %umax = call i32 @llvm.umax(i32 %0, i32 %rdx)
   %iv.next = add nuw nsw i64 %iv, 1
   %exitcond.not = icmp eq i64 %iv.next, %n
   br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll
index 53b0e95395869..a918e4c61425d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll
@@ -473,8 +473,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
-; IF-EVL-NEXT:    [[TMP13:%.*]] = icmp slt <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_PHI]]
-; IF-EVL-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
+; IF-EVL-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.smin.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
 ; IF-EVL-NEXT:    [[TMP15]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP14]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
@@ -505,8 +504,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
 ; NO-VP-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
-; NO-VP-NEXT:    [[TMP9:%.*]] = icmp slt <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; NO-VP-NEXT:    [[TMP10]] = select <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
+; NO-VP-NEXT:    [[TMP10]] = call <vscale x 4 x i32> @llvm.smin.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
 ; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
@@ -523,8 +521,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SMIN:%.*]], [[FOR_BODY]] ]
 ; NO-VP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; NO-VP-NEXT:    [[CMP_I:%.*]] = icmp slt i32 [[TMP13]], [[RDX]]
-; NO-VP-NEXT:    [[SMIN]] = select i1 [[CMP_I]], i32 [[TMP13]], i32 [[RDX]]
+; NO-VP-NEXT:    [[SMIN]] = call i32 @llvm.smin.i32(i32 [[TMP13]], i32 [[RDX]])
 ; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
@@ -540,8 +537,7 @@ for.body:
   %rdx = phi i32 [ %start, %entry ], [ %smin, %for.body ]
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv
   %0 = load i32, ptr %arrayidx, align 4
-  %cmp.i = icmp slt i32 %0, %rdx
-  %smin = select i1 %cmp.i, i32 %0, i32 %rdx
+  %smin = call i32 @llvm.smin.i32(i32 %0, i32 %rdx)
   %iv.next = add nuw nsw i64 %iv, 1
   %exitcond.not = icmp eq i64 %iv.next, %n
   br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0
@@ -565,8 +561,7 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
-; IF-EVL-NEXT:    [[TMP13:%.*]] = icmp sgt <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_PHI]]
-; IF-EVL-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
+; IF-EVL-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
 ; IF-EVL-NEXT:    [[TMP15]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP14]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
@@ -597,8 +592,7 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
 ; NO-VP-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
-; NO-VP-NEXT:    [[TMP9:%.*]] = icmp sgt <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; NO-VP-NEXT:    [[TMP10]] = select <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
+; NO-VP-NEXT:    [[TMP10]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
 ; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
@@ -615,8 +609,7 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SMAX:%.*]], [[FOR_BODY]] ]
 ; NO-VP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; NO-VP-NEXT:    [[CMP_I:%.*]] = icmp sgt i32 [[TMP13]], [[RDX]]
-; NO-VP-NEXT:    [[SMAX]] = select i1 [[CMP_I]], i32 [[TMP13]], i32 [[RDX]]
+; NO-VP-NEXT:    [[SMAX]] = call i32 @llvm.smax.i32(i32 [[TMP13]], i32 [[RDX]])
 ; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
@@ -632,8 +625,7 @@ for.body:
   %rdx = phi i32 [ %start, %entry ], [ %smax, %for.body ]
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv
   %0 = load i32, ptr %arrayidx, align 4
-  %cmp.i = icmp sgt i32 %0, %rdx
-  %smax = select i1 %cmp.i, i32 %0, i32 %rdx
+  %smax = call i32 @llvm.smax.i32(i32 %0, i32 %rdx)
   %iv.next = add nuw nsw i64 %iv, 1
   %exitcond.not = icmp eq i64 %iv.next, %n
   br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0
@@ -657,8 +649,7 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
-; IF-EVL-NEXT:    [[TMP13:%.*]] = icmp ult <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_PHI]]
-; IF-EVL-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
+; IF-EVL-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.umin.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
 ; IF-EVL-NEXT:    [[TMP15]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP14]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
@@ -689,8 +680,7 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
 ; NO-VP-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
-; NO-VP-NEXT:    [[TMP9:%.*]] = icmp ult <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; NO-VP-NEXT:    [[TMP10]] = select <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
+; NO-VP-NEXT:    [[TMP10]] = call <vscale x 4 x i32> @llvm.umin.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
 ; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
@@ -707,8 +697,7 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[UMIN:%.*]], [[FOR_BODY]] ]
 ; NO-VP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; NO-VP-NEXT:    [[CMP_I:%.*]] = icmp ult i32 [[TMP13]], [[RDX]]
-; NO-VP-NEXT:    [[UMIN]] = select i1 [[CMP_I]], i32 [[TMP13]], i32 [[RDX]]
+; NO-VP-NEXT:    [[UMIN]] = call i32 @llvm.umin.i32(i32 [[TMP13]], i32 [[RDX]])
 ; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
@@ -724,8 +713,7 @@ for.body:
   %rdx = phi i32 [ %start, %entry ], [ %umin, %for.body ]
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv
   %0 = load i32, ptr %arrayidx, align 4
-  %cmp.i = icmp ult i32 %0, %rdx
-  %umin = select i1 %cmp.i, i32 %0, i32 %rdx
+  %umin = call i32 @llvm.umin.i32(i32 %0, i32 %rdx)
   %iv.next = add nuw nsw i64 %iv, 1
   %exitcond.not = icmp eq i64 %iv.next, %n
   br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0
@@ -749,8 +737,7 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
-; IF-EVL-NEXT:    [[TMP13:%.*]] = icmp ugt <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_PHI]]
-; IF-EVL-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
+; IF-EVL-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.umax.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
 ; IF-EVL-NEXT:    [[TMP15]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP14]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
@@ -781,8 +768,7 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
 ; NO-VP-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
-; NO-VP-NEXT:    [[TMP9:%.*]] = icmp ugt <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; NO-VP-NEXT:    [[TMP10]] = select <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
+; NO-VP-NEXT:    [[TMP10]] = call <vscale x 4 x i32> @llvm.umax.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]])
 ; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
@@ -799,8 +785,7 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[UMAX:%.*]], [[FOR_BODY]] ]
 ; NO-VP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; NO-VP-NEXT:    [[CMP_I:%.*]] = icmp ugt i32 [[TMP13]], [[RDX]]
-; NO-VP-NEXT:    [[UMAX]] = select i1 [[CMP_I]], i32 [[TMP13]], i32 [[RDX]]
+; NO-VP-NEXT:    [[UMAX]] = call i32 @llvm.umax.i32(i32 [[TMP13]], i32 [[RDX]])
 ; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
@@ -816,8 +801,7 @@ for.body:
   %rdx = phi i32 [ %start, %entry ], [ %umax, %for.body ]
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv
   %0 = load i32, ptr %arrayidx, align 4
-  %cmp.i = icmp ugt i32 %0, %rdx
-  %umax = select i1 %cmp.i, i32 %0, i32 %rdx
+  %umax = call i32 @llvm.umax.i32(i32 %0, i32 %rdx)
   %iv.next = add nuw nsw i64 %iv, 1
   %exitcond.not = icmp eq i64 %iv.next, %n
   br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions-tail-folded.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions-tail-folded.ll
index 95a6601c4a2a2..beba6a437d880 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions-tail-folded.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions-tail-folded.ll
@@ -1256,7 +1256,7 @@ define i32 @print_umax_reduction(ptr %y) {
 ; CHECK-NEXT:      CLONE ir<%gep> = getelementptr inbounds ir<%y>, vp<[[VP4]]>
 ; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%lv> = load vp<[[VP5]]>
-; CHECK-NEXT:      REDUCE ir<%red.next> = ir<%red> +  reduce.umax (ir<%lv>)
+; CHECK-NEXT:      WIDEN-INTRINSIC ir<%red.next> = call llvm.umax(ir<%red>, ir<%lv>)
 ; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
 ; CHECK-NEXT:      EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
 ; CHECK-NEXT:    No successors
@@ -1264,7 +1264,7 @@ define i32 @print_umax_reduction(ptr %y) {
 ; CHECK-NEXT:  Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  middle.block:
-; CHECK-NEXT:    EMIT vp<[[VP7:%[0-9]+]]> = compute-reduction-result (umax, in-loop) ir<%red.next>
+; CHECK-NEXT:    EMIT vp<[[VP7:%[0-9]+]]> = compute-reduction-result (umax) ir<%red.next>
 ; CHECK-NEXT:    EMIT vp<%cmp.n> = icmp eq ir<100>, vp<[[VP2]]>
 ; CHECK-NEXT:    EMIT branch-on-cond vp<%cmp.n>
 ; CHECK-NEXT:  Successor(s): ir-bb<exit>, scalar.ph
@@ -1283,8 +1283,7 @@ loop:
   %red = phi i32 [ 0, %entry ], [ %red.next, %loop ]
   %gep = getelementptr inbounds i32, ptr %y, i64 %iv
   %lv = load i32, ptr %gep, align 4
-  %icmp = icmp ult i32 %lv, %red
-  %red.next = select i1 %icmp, i32 %red, i32 %lv
+  %red.next = call i32 @llvm.umax(i32 %red, i32 %lv)
   %iv.next = add i64 %iv, 1
   %ec = icmp eq i64 %iv.next, 100
   br i1 %ec, label %exit, label %loop
@@ -1316,7 +1315,7 @@ define i32 @print_umin_reduction(ptr %y) {
 ; CHECK-NEXT:      CLONE ir<%gep> = getelementptr inbounds ir<%y>, vp<[[VP4]]>
 ; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%lv> = load vp<[[VP5]]>
-; CHECK-NEXT:      REDUCE ir<%red.next> = ir<%red> +  reduce.umin (ir<%lv>)
+; CHECK-NEXT:      WIDEN-INTRINSIC ir<%red.next> = call llvm.umin(ir<%lv>, ir<%red>)
 ; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
 ; CHECK-NEXT:      EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
 ; CHECK-NEXT:    No successors
@@ -1324,7 +1323,7 @@ define i32 @print_umin_reduction(ptr %y) {
 ; CHECK-NEXT:  Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  middle.block:
-; CHECK-NEXT:    EMIT vp<[[VP7:%[0-9]+]]> = compute-reduction-result (umin, in-loop) ir<%red.next>
+; CHECK-NEXT:    EMIT vp<[[VP7:%[0-9]+]]> = compute-reduction-result (umin) ir<%red.next>
 ; CHECK-NEXT:    EMIT vp<%cmp.n> = icmp eq ir<100>, vp<[[VP2]]>
 ; CHECK-NEXT:    EMIT branch-on-cond vp<%cmp.n>
 ; CHECK-NEXT:  Successor(s): ir-bb<exit>, scalar.ph
@@ -1343,8 +1342,7 @@ loop:
   %red = phi i32 [ 0, %entry ], [ %red.next, %loop ]
   %gep = getelementptr inbounds i32, ptr %y, i64 %iv
   %lv = load i32, ptr %gep, align 4
-  %icmp = icmp ult i32 %lv, %red
-  %red.next = select i1 %icmp, i32 %lv, i32 %red
+  %red.next = call i32 @llvm.umin(i32 %lv, i32 %red)
   %iv.next = add i64 %iv, 1
   %ec = icmp eq i64 %iv.next, 100
   br i1 %ec, label %exit, label %loop
@@ -1376,7 +1374,7 @@ define i32 @print_smax_reduction(ptr %y) {
 ; CHECK-NEXT:      CLONE ir<%gep> = getelementptr inbounds ir<%y>, vp<[[VP4]]>
 ; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%lv> = load vp<[[VP5]]>
-; CHECK-NEXT:      REDUCE ir<%red.next> = ir<%red> +  reduce.smax (ir<%lv>)
+; CHECK-NEXT:      WIDEN-INTRINSIC ir<%red.next> = call llvm.smax(ir<%red>, ir<%lv>)
 ; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
 ; CHECK-NEXT:      EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
 ; CHECK-NEXT:    No successors
@@ -1384,7 +1382,7 @@ define i32 @print_smax_reduction(ptr %y) {
 ; CHECK-NEXT:  Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  middle.block:
-; CHECK-NEXT:    EMIT vp<[[VP7:%[0-9]+]]> = compute-reduction-result (smax, in-loop) ir<%red.next>
+; CHECK-NEXT:    EMIT vp<[[VP7:%[0-9]+]]> = compute-reduction-result (smax) ir<%red.next>
 ; CHECK-NEXT:    EMIT vp<%cmp.n> = icmp eq ir<100>, vp<[[VP2]]>
 ; CHECK-NEXT:    EMIT branch-on-cond vp<%cmp.n>
 ; CHECK-NEXT:  Successor(s): ir-bb<exit>, scalar.ph
@@ -1403,8 +1401,7 @@ loop:
   %red = phi i32 [ 0, %entry ], [ %red.next, %loop ]
   %gep = getelementptr inbounds i32, ptr %y, i64 %iv
   %lv = load i32, ptr %gep, align 4
-  %icmp = icmp slt i32 %lv, %red
-  %red.next = select i1 %icmp, i32 %red, i32 %lv
+  %red.next = call i32 @llvm.smax(i32 %red, i32 %lv)
   %iv.next = add i64 %iv, 1
   %ec = icmp eq i64 %iv.next, 100
   br i1 %ec, label %exit, label %loop
@@ -1436,7 +1433,7 @@ define i32 @print_smin_reduction(ptr %y) {
 ; CHECK-NEXT:      CLONE ir<%gep> = getelementptr inbounds ir<%y>, vp<[[VP4]]>
 ; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%lv> = load vp<[[VP5]]>
-; CHECK-NEXT:      REDUCE ir<%red.next> = ir<%red> +  reduce.smin (ir<%lv>)
+; CHECK-NEXT:      WIDEN-INTRINSIC ir<%red.next> = call llvm.smin(ir<%lv>, ir<%red>)
 ; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
 ; CHECK-NEXT:      EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
 ; CHECK-NEXT:    No successors
@@ -1444,7 +1441,7 @@ define i32 @print_smin_reduction(ptr %y) {
 ; CHECK-NEXT:  Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  middle.block:
-; CHECK-NEXT:    EMIT vp<[[VP7:%[0-9]+]]> = compute-reduction-result (smin, in-loop) ir<%red.next>
+; CHECK-NEXT:    EMIT vp<[[VP7:%[0-9]+]]> = compute-reduction-result (smin) ir<%red.next>
 ; CHECK-NEXT:    EMIT vp<%cmp.n> = icmp eq ir<100>, vp<[[VP2]]>
 ; CHECK-NEXT:    EMIT branch-on-cond vp<%cmp.n>
 ; CHECK-NEXT:  Successor(s): ir-bb<exit>, scalar.ph
@@ -1463,8 +1460,7 @@ loop:
   %red = phi i32 [ 0, %entry ], [ %red.next, %loop ]
   %gep = getelementptr inbounds i32, ptr %y, i64 %iv
   %lv = load i32, ptr %gep, align 4
-  %icmp = icmp slt i32 %lv, %red
-  %red.next = select i1 %icmp, i32 %lv, i32 %red
+  %red.next = call i32 @llvm.smin(i32 %lv, i32 %red)
   %iv.next = add i64 %iv, 1
   %ec = icmp eq i64 %iv.next, 100
   br i1 %ec, label %exit, label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll
index e6bddd4d18a67..1aee414ec7ee1 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll
@@ -26,7 +26,7 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%red> = phi (fadd) fast vp<[[VP3]]>, ir<%red.next>
 ; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[VP5]]>
-; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
+; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%lv> = load vp<[[VP6]]>
 ; CHECK-NEXT:      REDUCE ir<%red.next> = ir<%red> + fast  reduce.fadd (ir<%lv>)
 ; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
@@ -100,7 +100,7 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%red> = phi (fadd) fast vp<[[VP3]]>, ir<%red.next>
 ; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[VP5]]>
-; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
+; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%lv> = load vp<[[VP6]]>
 ; CHECK-NEXT:      REDUCE ir<%red.next> = ir<%red> + fast  reduce.fadd (ir<%lv>)
 ; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
@@ -176,10 +176,10 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%sum.07> = phi (fmuladd) nnan ninf nsz vp<[[VP3]]>, ir<%muladd>
 ; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP5]]>
-; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
+; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%l.a> = load vp<[[VP6]]>
 ; CHECK-NEXT:      CLONE ir<%arrayidx2> = getelementptr inbounds ir<%b>, vp<[[VP5]]>
-; CHECK-NEXT:      vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx2>
+; CHECK-NEXT:      vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx2>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%l.b> = load vp<[[VP7]]>
 ; CHECK-NEXT:      EMIT vp<[[VP8:%[0-9]+]]> = fmul nnan ninf nsz ir<%l.a>, ir<%l.b>
 ; CHECK-NEXT:      REDUCE ir<%muladd> = ir<%sum.07> + nnan ninf nsz  reduce.fmuladd (vp<[[VP8]]>)
@@ -259,7 +259,7 @@ define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) {
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%rdx> = phi (find-iv) ir<-9223372036854775808>, ir<%cond>
 ; CHECK-NEXT:      vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      CLONE ir<%gep.a> = getelementptr inbounds ir<%a>, vp<[[VP4]]>
-; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep.a>
+; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep.a>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%l.a> = load vp<[[VP5]]>
 ; CHECK-NEXT:      WIDEN ir<%cmp2> = icmp eq ir<%l.a>, ir<%start>
 ; CHECK-NEXT:      WIDEN ir<%cond> = select ir<%cmp2>, ir<%iv>, ir<%rdx>
@@ -338,7 +338,7 @@ define i64 @print_extended_reduction(ptr nocapture readonly %x, ptr nocapture re
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP3]]>, vp<[[VP7:%[0-9]+]]>
 ; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP5]]>
-; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
+; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%load0> = load vp<[[VP6]]>
 ; CHECK-NEXT:      EXPRESSION vp<[[VP7]]> = ir<%rdx> + reduce.add (ir<%load0> zext to i64)
 ; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
@@ -415,10 +415,10 @@ define i64 @print_mulacc(ptr nocapture readonly %x, ptr nocapture readonly %y, i
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]>
 ; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP5]]>
-; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
+; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%load0> = load vp<[[VP6]]>
 ; CHECK-NEXT:      CLONE ir<%arrayidx1> = getelementptr inbounds ir<%y>, vp<[[VP5]]>
-; CHECK-NEXT:      vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx1>
+; CHECK-NEXT:      vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx1>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%load1> = load vp<[[VP7]]>
 ; CHECK-NEXT:      EXPRESSION vp<[[VP8]]> = ir<%rdx> + reduce.add (mul nsw ir<%load0>, ir<%load1>)
 ; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
@@ -499,10 +499,10 @@ define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture reado
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]>
 ; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP5]]>
-; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
+; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%load0> = load vp<[[VP6]]>
 ; CHECK-NEXT:      CLONE ir<%arrayidx1> = getelementptr inbounds ir<%y>, vp<[[VP5]]>
-; CHECK-NEXT:      vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx1>
+; CHECK-NEXT:      vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx1>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%load1> = load vp<[[VP7]]>
 ; CHECK-NEXT:      EXPRESSION vp<[[VP8]]> = ir<%rdx> + reduce.add (mul nsw (ir<%load0> sext to i64), (ir<%load1> sext to i64))
 ; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
@@ -589,7 +589,7 @@ define i64 @print_extended_sub_reduction(ptr nocapture readonly %x, ptr nocaptur
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%rdx> = phi (sub) vp<[[VP3]]>, vp<[[VP7:%[0-9]+]]>
 ; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP5]]>
-; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
+; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%load0> = load vp<[[VP6]]>
 ; CHECK-NEXT:      EXPRESSION vp<[[VP7]]> = ir<%rdx> + reduce.sub (ir<%load0> zext to i64)
 ; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
@@ -666,10 +666,10 @@ define i32 @print_mulacc_sub(ptr %a, ptr %b) {
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%accum> = phi (sub) vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]>
 ; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[VP5]]>
-; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.a>
+; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.a>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%load.a> = load vp<[[VP6]]>
 ; CHECK-NEXT:      CLONE ir<%gep.b> = getelementptr ir<%b>, vp<[[VP5]]>
-; CHECK-NEXT:      vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.b>
+; CHECK-NEXT:      vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.b>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%load.b> = load vp<[[VP7]]>
 ; CHECK-NEXT:      EXPRESSION vp<[[VP8]]> = ir<%accum> + reduce.sub (mul (ir<%load.b> zext to i32), (ir<%load.a> zext to i32))
 ; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
@@ -753,10 +753,10 @@ define i32 @print_mulacc_negated(ptr %a, ptr %b) {
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%accum> = phi (add) vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]>
 ; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[VP5]]>
-; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.a>
+; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.a>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%load.a> = load vp<[[VP6]]>
 ; CHECK-NEXT:      CLONE ir<%gep.b> = getelementptr ir<%b>, vp<[[VP5]]>
-; CHECK-NEXT:      vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.b>
+; CHECK-NEXT:      vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.b>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%load.b> = load vp<[[VP7]]>
 ; CHECK-NEXT:      EXPRESSION vp<[[VP8]]> = ir<%accum> + reduce.add (sub (0, mul (ir<%load.b> zext to i32), (ir<%load.a> zext to i32)))
 ; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
@@ -842,10 +842,10 @@ define i64 @print_mulacc_sub_extended(ptr nocapture readonly %x, ptr nocapture r
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%rdx> = phi (sub) vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]>
 ; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP5]]>
-; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
+; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%load0> = load vp<[[VP6]]>
 ; CHECK-NEXT:      CLONE ir<%arrayidx1> = getelementptr inbounds ir<%y>, vp<[[VP5]]>
-; CHECK-NEXT:      vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx1>
+; CHECK-NEXT:      vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx1>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%load1> = load vp<[[VP7]]>
 ; CHECK-NEXT:      EXPRESSION vp<[[VP8]]> = ir<%rdx> + reduce.sub (mul nsw (ir<%load0> sext to i64), (ir<%load1> sext to i64))
 ; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
@@ -932,7 +932,7 @@ define i64 @print_mulacc_duplicate_extends(ptr nocapture readonly %x, ptr nocapt
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%rdx> = phi (sub) vp<[[VP3]]>, vp<[[VP7:%[0-9]+]]>
 ; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP5]]>
-; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
+; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%load0> = load vp<[[VP6]]>
 ; CHECK-NEXT:      EXPRESSION vp<[[VP7]]> = ir<%rdx> + reduce.sub (mul nsw (ir<%load0> sext to i64), (ir<%load0> sext to i64))
 ; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
@@ -1015,7 +1015,7 @@ define i32 @print_mulacc_extended_const(ptr %start, ptr %end) {
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP5]]>, vp<[[VP9:%[0-9]+]]>
 ; CHECK-NEXT:      vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP7]]>
-; CHECK-NEXT:      vp<[[VP8:%[0-9]+]]> = vector-pointer vp<%next.gep>
+; CHECK-NEXT:      vp<[[VP8:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%l> = load vp<[[VP8]]>
 ; CHECK-NEXT:      EXPRESSION vp<[[VP9]]> = ir<%red> + reduce.add (mul (ir<%l> zext to i32), (ir<63> zext to i32))
 ; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP6]]>, vp<[[VP1]]>
@@ -1093,7 +1093,7 @@ define i32 @print_mulacc_extended_const_lhs(ptr %start, ptr %end) {
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP5]]>, vp<[[VP9:%[0-9]+]]>
 ; CHECK-NEXT:      vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP7]]>
-; CHECK-NEXT:      vp<[[VP8:%[0-9]+]]> = vector-pointer vp<%next.gep>
+; CHECK-NEXT:      vp<[[VP8:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%l> = load vp<[[VP8]]>
 ; CHECK-NEXT:      WIDEN-CAST ir<%l.ext> = zext ir<%l> to i32
 ; CHECK-NEXT:      EXPRESSION vp<[[VP9]]> = ir<%red> + reduce.add (mul ir<63>, ir<%l.ext>)
@@ -1173,7 +1173,7 @@ define i32 @print_mulacc_not_extended_const(ptr %start, ptr %end) {
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP5]]>, vp<[[VP9:%[0-9]+]]>
 ; CHECK-NEXT:      vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP7]]>
-; CHECK-NEXT:      vp<[[VP8:%[0-9]+]]> = vector-pointer vp<%next.gep>
+; CHECK-NEXT:      vp<[[VP8:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%l> = load vp<[[VP8]]>
 ; CHECK-NEXT:      WIDEN-CAST ir<%l.ext> = sext ir<%l> to i32
 ; CHECK-NEXT:      EXPRESSION vp<[[VP9]]> = ir<%red> + reduce.add (mul ir<%l.ext>, ir<128>)
@@ -1253,7 +1253,7 @@ define i64 @print_ext_mulacc_extended_const(ptr %start, ptr %end) {
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP5]]>, vp<[[VP9:%[0-9]+]]>
 ; CHECK-NEXT:      vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP7]]>
-; CHECK-NEXT:      vp<[[VP8:%[0-9]+]]> = vector-pointer vp<%next.gep>
+; CHECK-NEXT:      vp<[[VP8:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%l> = load vp<[[VP8]]>
 ; CHECK-NEXT:      EXPRESSION vp<[[VP9]]> = ir<%red> + reduce.add (mul (ir<%l> zext to i64), (ir<63> zext to i64))
 ; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP6]]>, vp<[[VP1]]>
@@ -1334,7 +1334,7 @@ define i64 @print_ext_mulacc_not_extended_const(ptr %start, ptr %end) {
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP5]]>, vp<[[VP10:%[0-9]+]]>
 ; CHECK-NEXT:      vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP7]]>
-; CHECK-NEXT:      vp<[[VP8:%[0-9]+]]> = vector-pointer vp<%next.gep>
+; CHECK-NEXT:      vp<[[VP8:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%l> = load vp<[[VP8]]>
 ; CHECK-NEXT:      WIDEN-CAST ir<%l.ext> = sext ir<%l> to i32
 ; CHECK-NEXT:      EMIT vp<[[VP9:%[0-9]+]]> = shl ir<%l.ext>, ir<7>
@@ -1510,7 +1510,7 @@ define i32 @print_umax_reduction_out_of_loop(ptr %y) {
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%red> = phi (umax) ir<0>, ir<%red.next>
 ; CHECK-NEXT:      vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      CLONE ir<%gep> = getelementptr inbounds ir<%y>, vp<[[VP4]]>
-; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep>
+; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%lv> = load vp<[[VP5]]>
 ; CHECK-NEXT:      WIDEN-INTRINSIC ir<%red.next> = call llvm.umax(ir<%lv>, ir<%red>)
 ; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
@@ -1583,9 +1583,9 @@ define i32 @print_umax_reduction(ptr %y) {
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%red> = phi (umax) ir<0>, ir<%red.next>
 ; CHECK-NEXT:      vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      CLONE ir<%gep> = getelementptr inbounds ir<%y>, vp<[[VP4]]>
-; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep>
+; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%lv> = load vp<[[VP5]]>
-; CHECK-NEXT:      REDUCE ir<%red.next> = ir<%red> +  reduce.umax (ir<%lv>)
+; CHECK-NEXT:      WIDEN-INTRINSIC ir<%red.next> = call llvm.umax(ir<%red>, ir<%lv>)
 ; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
 ; CHECK-NEXT:      EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
 ; CHECK-NEXT:    No successors
@@ -1593,7 +1593,7 @@ define i32 @print_umax_reduction(ptr %y) {
 ; CHECK-NEXT:  Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  middle.block:
-; CHECK-NEXT:    EMIT vp<[[VP7:%[0-9]+]]> = compute-reduction-result (umax, in-loop) ir<%red.next>
+; CHECK-NEXT:    EMIT vp<[[VP7:%[0-9]+]]> = compute-reduction-result (umax) ir<%red.next>
 ; CHECK-NEXT:    EMIT vp<%cmp.n> = icmp eq ir<100>, vp<[[VP2]]>
 ; CHECK-NEXT:    EMIT branch-on-cond vp<%cmp.n>
 ; CHECK-NEXT:  Successor(s): ir-bb<exit>, scalar.ph
@@ -1612,8 +1612,7 @@ define i32 @print_umax_reduction(ptr %y) {
 ; CHECK-NEXT:    IR   %red = phi i32 [ 0, %entry ], [ %red.next, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
 ; CHECK-NEXT:    IR   %gep = getelementptr inbounds i32, ptr %y, i64 %iv
 ; CHECK-NEXT:    IR   %lv = load i32, ptr %gep, align 4
-; CHECK-NEXT:    IR   %icmp = icmp ult i32 %lv, %red
-; CHECK-NEXT:    IR   %red.next = select i1 %icmp, i32 %red, i32 %lv
+; CHECK-NEXT:    IR   %red.next = call i32 @llvm.umax.i32(i32 %red, i32 %lv)
 ; CHECK-NEXT:    IR   %iv.next = add i64 %iv, 1
 ; CHECK-NEXT:    IR   %ec = icmp eq i64 %iv.next, 100
 ; CHECK-NEXT:  No successors
@@ -1627,8 +1626,7 @@ loop:
   %red = phi i32 [ 0, %entry ], [ %red.next, %loop ]
   %gep = getelementptr inbounds i32, ptr %y, i64 %iv
   %lv = load i32, ptr %gep, align 4
-  %icmp = icmp ult i32 %lv, %red
-  %red.next = select i1 %icmp, i32 %red, i32 %lv
+  %red.next = call i32 @llvm.umax(i32 %red, i32 %lv)
   %iv.next = add i64 %iv, 1
   %ec = icmp eq i64 %iv.next, 100
   br i1 %ec, label %exit, label %loop
@@ -1658,9 +1656,9 @@ define i32 @print_umin_reduction(ptr %y) {
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%red> = phi (umin) ir<0>, ir<%red.next>
 ; CHECK-NEXT:      vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      CLONE ir<%gep> = getelementptr inbounds ir<%y>, vp<[[VP4]]>
-; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep>
+; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%lv> = load vp<[[VP5]]>
-; CHECK-NEXT:      REDUCE ir<%red.next> = ir<%red> +  reduce.umin (ir<%lv>)
+; CHECK-NEXT:      WIDEN-INTRINSIC ir<%red.next> = call llvm.umin(ir<%lv>, ir<%red>)
 ; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
 ; CHECK-NEXT:      EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
 ; CHECK-NEXT:    No successors
@@ -1668,7 +1666,7 @@ define i32 @print_umin_reduction(ptr %y) {
 ; CHECK-NEXT:  Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  middle.block:
-; CHECK-NEXT:    EMIT vp<[[VP7:%[0-9]+]]> = compute-reduction-result (umin, in-loop) ir<%red.next>
+; CHECK-NEXT:    EMIT vp<[[VP7:%[0-9]+]]> = compute-reduction-result (umin) ir<%red.next>
 ; CHECK-NEXT:    EMIT vp<%cmp.n> = icmp eq ir<100>, vp<[[VP2]]>
 ; CHECK-NEXT:    EMIT branch-on-cond vp<%cmp.n>
 ; CHECK-NEXT:  Successor(s): ir-bb<exit>, scalar.ph
@@ -1687,8 +1685,7 @@ define i32 @print_umin_reduction(ptr %y) {
 ; CHECK-NEXT:    IR   %red = phi i32 [ 0, %entry ], [ %red.next, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
 ; CHECK-NEXT:    IR   %gep = getelementptr inbounds i32, ptr %y, i64 %iv
 ; CHECK-NEXT:    IR   %lv = load i32, ptr %gep, align 4
-; CHECK-NEXT:    IR   %icmp = icmp ult i32 %lv, %red
-; CHECK-NEXT:    IR   %red.next = select i1 %icmp, i32 %lv, i32 %red
+; CHECK-NEXT:    IR   %red.next = call i32 @llvm.umin.i32(i32 %lv, i32 %red)
 ; CHECK-NEXT:    IR   %iv.next = add i64 %iv, 1
 ; CHECK-NEXT:    IR   %ec = icmp eq i64 %iv.next, 100
 ; CHECK-NEXT:  No successors
@@ -1702,8 +1699,7 @@ loop:
   %red = phi i32 [ 0, %entry ], [ %red.next, %loop ]
   %gep = getelementptr inbounds i32, ptr %y, i64 %iv
   %lv = load i32, ptr %gep, align 4
-  %icmp = icmp ult i32 %lv, %red
-  %red.next = select i1 %icmp, i32 %lv, i32 %red
+  %red.next = call i32 @llvm.umin(i32 %lv, i32 %red)
   %iv.next = add i64 %iv, 1
   %ec = icmp eq i64 %iv.next, 100
   br i1 %ec, label %exit, label %loop
@@ -1733,9 +1729,9 @@ define i32 @print_smax_reduction(ptr %y) {
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%red> = phi (smax) ir<0>, ir<%red.next>
 ; CHECK-NEXT:      vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      CLONE ir<%gep> = getelementptr inbounds ir<%y>, vp<[[VP4]]>
-; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep>
+; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%lv> = load vp<[[VP5]]>
-; CHECK-NEXT:      REDUCE ir<%red.next> = ir<%red> +  reduce.smax (ir<%lv>)
+; CHECK-NEXT:      WIDEN-INTRINSIC ir<%red.next> = call llvm.smax(ir<%red>, ir<%lv>)
 ; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
 ; CHECK-NEXT:      EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
 ; CHECK-NEXT:    No successors
@@ -1743,7 +1739,7 @@ define i32 @print_smax_reduction(ptr %y) {
 ; CHECK-NEXT:  Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  middle.block:
-; CHECK-NEXT:    EMIT vp<[[VP7:%[0-9]+]]> = compute-reduction-result (smax, in-loop) ir<%red.next>
+; CHECK-NEXT:    EMIT vp<[[VP7:%[0-9]+]]> = compute-reduction-result (smax) ir<%red.next>
 ; CHECK-NEXT:    EMIT vp<%cmp.n> = icmp eq ir<100>, vp<[[VP2]]>
 ; CHECK-NEXT:    EMIT branch-on-cond vp<%cmp.n>
 ; CHECK-NEXT:  Successor(s): ir-bb<exit>, scalar.ph
@@ -1762,8 +1758,7 @@ define i32 @print_smax_reduction(ptr %y) {
 ; CHECK-NEXT:    IR   %red = phi i32 [ 0, %entry ], [ %red.next, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
 ; CHECK-NEXT:    IR   %gep = getelementptr inbounds i32, ptr %y, i64 %iv
 ; CHECK-NEXT:    IR   %lv = load i32, ptr %gep, align 4
-; CHECK-NEXT:    IR   %icmp = icmp slt i32 %lv, %red
-; CHECK-NEXT:    IR   %red.next = select i1 %icmp, i32 %red, i32 %lv
+; CHECK-NEXT:    IR   %red.next = call i32 @llvm.smax.i32(i32 %red, i32 %lv)
 ; CHECK-NEXT:    IR   %iv.next = add i64 %iv, 1
 ; CHECK-NEXT:    IR   %ec = icmp eq i64 %iv.next, 100
 ; CHECK-NEXT:  No successors
@@ -1777,8 +1772,7 @@ loop:
   %red = phi i32 [ 0, %entry ], [ %red.next, %loop ]
   %gep = getelementptr inbounds i32, ptr %y, i64 %iv
   %lv = load i32, ptr %gep, align 4
-  %icmp = icmp slt i32 %lv, %red
-  %red.next = select i1 %icmp, i32 %red, i32 %lv
+  %red.next = call i32 @llvm.smax(i32 %red, i32 %lv)
   %iv.next = add i64 %iv, 1
   %ec = icmp eq i64 %iv.next, 100
   br i1 %ec, label %exit, label %loop
@@ -1808,9 +1802,9 @@ define i32 @print_smin_reduction(ptr %y) {
 ; CHECK-NEXT:      WIDEN-REDUCTION-PHI ir<%red> = phi (smin) ir<0>, ir<%red.next>
 ; CHECK-NEXT:      vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
 ; CHECK-NEXT:      CLONE ir<%gep> = getelementptr inbounds ir<%y>, vp<[[VP4]]>
-; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep>
+; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep>, ir<1>
 ; CHECK-NEXT:      WIDEN ir<%lv> = load vp<[[VP5]]>
-; CHECK-NEXT:      REDUCE ir<%red.next> = ir<%red> +  reduce.smin (ir<%lv>)
+; CHECK-NEXT:      WIDEN-INTRINSIC ir<%red.next> = call llvm.smin(ir<%lv>, ir<%red>)
 ; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
 ; CHECK-NEXT:      EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
 ; CHECK-NEXT:    No successors
@@ -1818,7 +1812,7 @@ define i32 @print_smin_reduction(ptr %y) {
 ; CHECK-NEXT:  Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  middle.block:
-; CHECK-NEXT:    EMIT vp<[[VP7:%[0-9]+]]> = compute-reduction-result (smin, in-loop) ir<%red.next>
+; CHECK-NEXT:    EMIT vp<[[VP7:%[0-9]+]]> = compute-reduction-result (smin) ir<%red.next>
 ; CHECK-NEXT:    EMIT vp<%cmp.n> = icmp eq ir<100>, vp<[[VP2]]>
 ; CHECK-NEXT:    EMIT branch-on-cond vp<%cmp.n>
 ; CHECK-NEXT:  Successor(s): ir-bb<exit>, scalar.ph
@@ -1837,8 +1831,7 @@ define i32 @print_smin_reduction(ptr %y) {
 ; CHECK-NEXT:    IR   %red = phi i32 [ 0, %entry ], [ %red.next, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
 ; CHECK-NEXT:    IR   %gep = getelementptr inbounds i32, ptr %y, i64 %iv
 ; CHECK-NEXT:    IR   %lv = load i32, ptr %gep, align 4
-; CHECK-NEXT:    IR   %icmp = icmp slt i32 %lv, %red
-; CHECK-NEXT:    IR   %red.next = select i1 %icmp, i32 %lv, i32 %red
+; CHECK-NEXT:    IR   %red.next = call i32 @llvm.smin.i32(i32 %lv, i32 %red)
 ; CHECK-NEXT:    IR   %iv.next = add i64 %iv, 1
 ; CHECK-NEXT:    IR   %ec = icmp eq i64 %iv.next, 100
 ; CHECK-NEXT:  No successors
@@ -1852,8 +1845,7 @@ loop:
   %red = phi i32 [ 0, %entry ], [ %red.next, %loop ]
   %gep = getelementptr inbounds i32, ptr %y, i64 %iv
   %lv = load i32, ptr %gep, align 4
-  %icmp = icmp slt i32 %lv, %red
-  %red.next = select i1 %icmp, i32 %lv, i32 %red
+  %red.next = call i32 @llvm.smin(i32 %lv, i32 %red)
   %iv.next = add i64 %iv, 1
   %ec = icmp eq i64 %iv.next, 100
   br i1 %ec, label %exit, label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
index a79636c0f1eef..100afc7049ebc 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
@@ -242,14 +242,10 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) {
 ; UNROLL-NO-IC-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> [[WIDE_LOAD2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; UNROLL-NO-IC-NEXT:    [[TMP7:%.*]] = sub nsw <4 x i32> [[WIDE_LOAD]], [[TMP5]]
 ; UNROLL-NO-IC-NEXT:    [[TMP8:%.*]] = sub nsw <4 x i32> [[WIDE_LOAD2]], [[TMP6]]
-; UNROLL-NO-IC-NEXT:    [[TMP9:%.*]] = icmp sgt <4 x i32> [[TMP7]], zeroinitializer
-; UNROLL-NO-IC-NEXT:    [[TMP10:%.*]] = icmp sgt <4 x i32> [[TMP8]], zeroinitializer
-; UNROLL-NO-IC-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[TMP9]], <4 x i32> [[TMP7]], <4 x i32> zeroinitializer
-; UNROLL-NO-IC-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[TMP10]], <4 x i32> [[TMP8]], <4 x i32> zeroinitializer
-; UNROLL-NO-IC-NEXT:    [[TMP13:%.*]] = icmp slt <4 x i32> [[VEC_PHI]], [[TMP11]]
-; UNROLL-NO-IC-NEXT:    [[TMP14:%.*]] = icmp slt <4 x i32> [[VEC_PHI1]], [[TMP12]]
-; UNROLL-NO-IC-NEXT:    [[TMP15]] = select <4 x i1> [[TMP13]], <4 x i32> [[VEC_PHI]], <4 x i32> [[TMP11]]
-; UNROLL-NO-IC-NEXT:    [[TMP16]] = select <4 x i1> [[TMP14]], <4 x i32> [[VEC_PHI1]], <4 x i32> [[TMP12]]
+; UNROLL-NO-IC-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[TMP7]], <4 x i32> zeroinitializer)
+; UNROLL-NO-IC-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[TMP8]], <4 x i32> zeroinitializer)
+; UNROLL-NO-IC-NEXT:    [[TMP15]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[TMP9]])
+; UNROLL-NO-IC-NEXT:    [[TMP16]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VEC_PHI1]], <4 x i32> [[TMP10]])
 ; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; UNROLL-NO-IC-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; UNROLL-NO-IC-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -277,10 +273,8 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) {
 ; UNROLL-NO-IC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
 ; UNROLL-NO-IC-NEXT:    [[TMP20]] = load i32, ptr [[ARRAYIDX]], align 4
 ; UNROLL-NO-IC-NEXT:    [[SUB3:%.*]] = sub nsw i32 [[TMP20]], [[TMP19]]
-; UNROLL-NO-IC-NEXT:    [[CMP4:%.*]] = icmp sgt i32 [[SUB3]], 0
-; UNROLL-NO-IC-NEXT:    [[COND:%.*]] = select i1 [[CMP4]], i32 [[SUB3]], i32 0
-; UNROLL-NO-IC-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[MINMAX_028]], [[COND]]
-; UNROLL-NO-IC-NEXT:    [[MINMAX_0_COND]] = select i1 [[CMP5]], i32 [[MINMAX_028]], i32 [[COND]]
+; UNROLL-NO-IC-NEXT:    [[COND:%.*]] = call i32 @llvm.smax.i32(i32 [[SUB3]], i32 0)
+; UNROLL-NO-IC-NEXT:    [[MINMAX_0_COND]] = call i32 @llvm.smin.i32(i32 [[MINMAX_028]], i32 [[COND]])
 ; UNROLL-NO-IC-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; UNROLL-NO-IC-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
 ; UNROLL-NO-IC-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
@@ -313,14 +307,10 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) {
 ; UNROLL-NO-VF-NEXT:    [[TMP6]] = load i32, ptr [[TMP4]], align 4
 ; UNROLL-NO-VF-NEXT:    [[TMP7:%.*]] = sub nsw i32 [[TMP5]], [[VECTOR_RECUR]]
 ; UNROLL-NO-VF-NEXT:    [[TMP8:%.*]] = sub nsw i32 [[TMP6]], [[TMP5]]
-; UNROLL-NO-VF-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP7]], 0
-; UNROLL-NO-VF-NEXT:    [[TMP10:%.*]] = icmp sgt i32 [[TMP8]], 0
-; UNROLL-NO-VF-NEXT:    [[TMP11:%.*]] = select i1 [[TMP9]], i32 [[TMP7]], i32 0
-; UNROLL-NO-VF-NEXT:    [[TMP12:%.*]] = select i1 [[TMP10]], i32 [[TMP8]], i32 0
-; UNROLL-NO-VF-NEXT:    [[TMP13:%.*]] = icmp slt i32 [[VEC_PHI]], [[TMP11]]
-; UNROLL-NO-VF-NEXT:    [[TMP14:%.*]] = icmp slt i32 [[VEC_PHI1]], [[TMP12]]
-; UNROLL-NO-VF-NEXT:    [[TMP15]] = select i1 [[TMP13]], i32 [[VEC_PHI]], i32 [[TMP11]]
-; UNROLL-NO-VF-NEXT:    [[TMP16]] = select i1 [[TMP14]], i32 [[VEC_PHI1]], i32 [[TMP12]]
+; UNROLL-NO-VF-NEXT:    [[TMP10:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP7]], i32 0)
+; UNROLL-NO-VF-NEXT:    [[TMP9:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP8]], i32 0)
+; UNROLL-NO-VF-NEXT:    [[TMP15]] = call i32 @llvm.smin.i32(i32 [[VEC_PHI]], i32 [[TMP10]])
+; UNROLL-NO-VF-NEXT:    [[TMP16]] = call i32 @llvm.smin.i32(i32 [[VEC_PHI1]], i32 [[TMP9]])
 ; UNROLL-NO-VF-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; UNROLL-NO-VF-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; UNROLL-NO-VF-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -346,10 +336,8 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) {
 ; UNROLL-NO-VF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
 ; UNROLL-NO-VF-NEXT:    [[TMP19]] = load i32, ptr [[ARRAYIDX]], align 4
 ; UNROLL-NO-VF-NEXT:    [[SUB3:%.*]] = sub nsw i32 [[TMP19]], [[TMP18]]
-; UNROLL-NO-VF-NEXT:    [[CMP4:%.*]] = icmp sgt i32 [[SUB3]], 0
-; UNROLL-NO-VF-NEXT:    [[COND:%.*]] = select i1 [[CMP4]], i32 [[SUB3]], i32 0
-; UNROLL-NO-VF-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[MINMAX_028]], [[COND]]
-; UNROLL-NO-VF-NEXT:    [[MINMAX_0_COND]] = select i1 [[CMP5]], i32 [[MINMAX_028]], i32 [[COND]]
+; UNROLL-NO-VF-NEXT:    [[COND:%.*]] = call i32 @llvm.smax.i32(i32 [[SUB3]], i32 0)
+; UNROLL-NO-VF-NEXT:    [[MINMAX_0_COND]] = call i32 @llvm.smin.i32(i32 [[MINMAX_028]], i32 [[COND]])
 ; UNROLL-NO-VF-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; UNROLL-NO-VF-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
 ; UNROLL-NO-VF-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
@@ -379,10 +367,8 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) {
 ; SINK-AFTER-NEXT:    [[WIDE_LOAD]] = load <4 x i32>, ptr [[TMP2]], align 4
 ; SINK-AFTER-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; SINK-AFTER-NEXT:    [[TMP5:%.*]] = sub nsw <4 x i32> [[WIDE_LOAD]], [[TMP4]]
-; SINK-AFTER-NEXT:    [[TMP6:%.*]] = icmp sgt <4 x i32> [[TMP5]], zeroinitializer
-; SINK-AFTER-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i32> [[TMP5]], <4 x i32> zeroinitializer
-; SINK-AFTER-NEXT:    [[TMP8:%.*]] = icmp slt <4 x i32> [[VEC_PHI]], [[TMP7]]
-; SINK-AFTER-NEXT:    [[TMP9]] = select <4 x i1> [[TMP8]], <4 x i32> [[VEC_PHI]], <4 x i32> [[TMP7]]
+; SINK-AFTER-NEXT:    [[TMP6:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[TMP5]], <4 x i32> zeroinitializer)
+; SINK-AFTER-NEXT:    [[TMP9]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[TMP6]])
 ; SINK-AFTER-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; SINK-AFTER-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; SINK-AFTER-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -409,10 +395,8 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) {
 ; SINK-AFTER-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
 ; SINK-AFTER-NEXT:    [[TMP13]] = load i32, ptr [[ARRAYIDX]], align 4
 ; SINK-AFTER-NEXT:    [[SUB3:%.*]] = sub nsw i32 [[TMP13]], [[TMP12]]
-; SINK-AFTER-NEXT:    [[CMP4:%.*]] = icmp sgt i32 [[SUB3]], 0
-; SINK-AFTER-NEXT:    [[COND:%.*]] = select i1 [[CMP4]], i32 [[SUB3]], i32 0
-; SINK-AFTER-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[MINMAX_028]], [[COND]]
-; SINK-AFTER-NEXT:    [[MINMAX_0_COND]] = select i1 [[CMP5]], i32 [[MINMAX_028]], i32 [[COND]]
+; SINK-AFTER-NEXT:    [[COND:%.*]] = call i32 @llvm.smax.i32(i32 [[SUB3]], i32 0)
+; SINK-AFTER-NEXT:    [[MINMAX_0_COND]] = call i32 @llvm.smin.i32(i32 [[MINMAX_028]], i32 [[COND]])
 ; SINK-AFTER-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; SINK-AFTER-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
 ; SINK-AFTER-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
@@ -442,10 +426,8 @@ scalar.body:
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
   %1 = load i32, ptr %arrayidx, align 4
   %sub3 = sub nsw i32 %1, %0
-  %cmp4 = icmp sgt i32 %sub3, 0
-  %cond = select i1 %cmp4, i32 %sub3, i32 0
-  %cmp5 = icmp slt i32 %minmax.028, %cond
-  %minmax.0.cond = select i1 %cmp5, i32 %minmax.028, i32 %cond
+  %cond = call i32 @llvm.smax(i32 %sub3, i32 0)
+  %minmax.0.cond = call i32 @llvm.smin(i32 %minmax.028, i32 %cond)
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp eq i32 %lftr.wideiv, %n
@@ -3351,12 +3333,12 @@ define void @unused_recurrence(ptr %a, i16 %n) {
 ; UNROLL-NO-IC-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
 ; UNROLL-NO-IC-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
 ; UNROLL-NO-IC:       [[SCALAR_PH]]:
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ -27, %[[ENTRY]] ]
-; UNROLL-NO-IC-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; UNROLL-NO-IC-NEXT:    [[TMP9:%.*]] = phi i16 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ -27, %[[ENTRY]] ]
+; UNROLL-NO-IC-NEXT:    [[TMP10:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; UNROLL-NO-IC-NEXT:    br label %[[FOR_COND:.*]]
 ; UNROLL-NO-IC:       [[FOR_COND]]:
-; UNROLL-NO-IC-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_COND]] ]
-; UNROLL-NO-IC-NEXT:    [[REC_1:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[REC_1_PREV:%.*]], %[[FOR_COND]] ]
+; UNROLL-NO-IC-NEXT:    [[IV:%.*]] = phi i16 [ [[TMP9]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_COND]] ]
+; UNROLL-NO-IC-NEXT:    [[REC_1:%.*]] = phi i16 [ [[TMP10]], %[[SCALAR_PH]] ], [ [[REC_1_PREV:%.*]], %[[FOR_COND]] ]
 ; UNROLL-NO-IC-NEXT:    [[USE_REC_1:%.*]] = sub i16 [[REC_1]], 10
 ; UNROLL-NO-IC-NEXT:    [[IV_NEXT]] = add i16 [[IV]], 1
 ; UNROLL-NO-IC-NEXT:    [[REC_1_PREV]] = add i16 [[IV_NEXT]], 5
@@ -3393,12 +3375,12 @@ define void @unused_recurrence(ptr %a, i16 %n) {
 ; UNROLL-NO-VF-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP6]], [[N_VEC]]
 ; UNROLL-NO-VF-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
 ; UNROLL-NO-VF:       [[SCALAR_PH]]:
-; UNROLL-NO-VF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ -27, %[[ENTRY]] ]
-; UNROLL-NO-VF-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; UNROLL-NO-VF-NEXT:    [[TMP11:%.*]] = phi i16 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ -27, %[[ENTRY]] ]
+; UNROLL-NO-VF-NEXT:    [[TMP12:%.*]] = phi i16 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; UNROLL-NO-VF-NEXT:    br label %[[FOR_COND:.*]]
 ; UNROLL-NO-VF:       [[FOR_COND]]:
-; UNROLL-NO-VF-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_COND]] ]
-; UNROLL-NO-VF-NEXT:    [[REC_1:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[REC_1_PREV:%.*]], %[[FOR_COND]] ]
+; UNROLL-NO-VF-NEXT:    [[IV:%.*]] = phi i16 [ [[TMP11]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_COND]] ]
+; UNROLL-NO-VF-NEXT:    [[REC_1:%.*]] = phi i16 [ [[TMP12]], %[[SCALAR_PH]] ], [ [[REC_1_PREV:%.*]], %[[FOR_COND]] ]
 ; UNROLL-NO-VF-NEXT:    [[USE_REC_1:%.*]] = sub i16 [[REC_1]], 10
 ; UNROLL-NO-VF-NEXT:    [[IV_NEXT]] = add i16 [[IV]], 1
 ; UNROLL-NO-VF-NEXT:    [[REC_1_PREV]] = add i16 [[IV_NEXT]], 5
@@ -3435,12 +3417,12 @@ define void @unused_recurrence(ptr %a, i16 %n) {
 ; SINK-AFTER-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
 ; SINK-AFTER-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
 ; SINK-AFTER:       [[SCALAR_PH]]:
-; SINK-AFTER-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ -27, %[[ENTRY]] ]
-; SINK-AFTER-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; SINK-AFTER-NEXT:    [[TMP9:%.*]] = phi i16 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ -27, %[[ENTRY]] ]
+; SINK-AFTER-NEXT:    [[TMP10:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; SINK-AFTER-NEXT:    br label %[[FOR_COND:.*]]
 ; SINK-AFTER:       [[FOR_COND]]:
-; SINK-AFTER-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_COND]] ]
-; SINK-AFTER-NEXT:    [[REC_1:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[REC_1_PREV:%.*]], %[[FOR_COND]] ]
+; SINK-AFTER-NEXT:    [[IV:%.*]] = phi i16 [ [[TMP9]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_COND]] ]
+; SINK-AFTER-NEXT:    [[REC_1:%.*]] = phi i16 [ [[TMP10]], %[[SCALAR_PH]] ], [ [[REC_1_PREV:%.*]], %[[FOR_COND]] ]
 ; SINK-AFTER-NEXT:    [[USE_REC_1:%.*]] = sub i16 [[REC_1]], 10
 ; SINK-AFTER-NEXT:    [[IV_NEXT]] = add i16 [[IV]], 1
 ; SINK-AFTER-NEXT:    [[REC_1_PREV]] = add i16 [[IV_NEXT]], 5
diff --git a/llvm/test/Transforms/LoopVectorize/minmax-extra-outside-users.ll b/llvm/test/Transforms/LoopVectorize/minmax-extra-outside-users.ll
index 1c58544bea456..55cace5738248 100644
--- a/llvm/test/Transforms/LoopVectorize/minmax-extra-outside-users.ll
+++ b/llvm/test/Transforms/LoopVectorize/minmax-extra-outside-users.ll
@@ -16,8 +16,7 @@ define i32 @smin_three_exit_users(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 2147483647), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT:    [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP2]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -34,8 +33,7 @@ define i32 @smin_three_exit_users(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[MIN:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[MIN]], [[V]]
-; CHECK-NEXT:    [[MIN_NEXT]] = select i1 [[CMP]], i32 [[MIN]], i32 [[V]]
+; CHECK-NEXT:    [[MIN_NEXT]] = call i32 @llvm.smin.i32(i32 [[MIN]], i32 [[V]])
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
@@ -55,8 +53,7 @@ loop:
   %min = phi i32 [ 2147483647, %entry ], [ %min.next, %loop ]
   %gep = getelementptr inbounds i32, ptr %src, i64 %iv
   %v = load i32, ptr %gep, align 4
-  %cmp = icmp slt i32 %min, %v
-  %min.next = select i1 %cmp, i32 %min, i32 %v
+  %min.next = call i32 @llvm.smin(i32 %min, i32 %v)
   %iv.next = add nuw nsw i64 %iv, 1
   %ec = icmp eq i64 %iv.next, %n
   br i1 %ec, label %exit, label %loop
@@ -85,8 +82,7 @@ define i32 @smax_three_exit_users(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -2147483648), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT:    [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP2]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -103,8 +99,7 @@ define i32 @smax_three_exit_users(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[MAX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[MAX]], [[V]]
-; CHECK-NEXT:    [[MAX_NEXT]] = select i1 [[CMP]], i32 [[MAX]], i32 [[V]]
+; CHECK-NEXT:    [[MAX_NEXT]] = call i32 @llvm.smax.i32(i32 [[MAX]], i32 [[V]])
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -124,8 +119,7 @@ loop:
   %max = phi i32 [ -2147483648, %entry ], [ %max.next, %loop ]
   %gep = getelementptr inbounds i32, ptr %src, i64 %iv
   %v = load i32, ptr %gep, align 4
-  %cmp = icmp sgt i32 %max, %v
-  %max.next = select i1 %cmp, i32 %max, i32 %v
+  %max.next = call i32 @llvm.smax(i32 %max, i32 %v)
   %iv.next = add nuw nsw i64 %iv, 1
   %ec = icmp eq i64 %iv.next, %n
   br i1 %ec, label %exit, label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
index bf6f1f89e2e6e..bc6ce82606cf2 100644
--- a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
@@ -13,8 +13,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 ; CHECK-LABEL: @max_red(
 ; CHECK: %[[VAR:.*]] = insertelement <2 x i32> poison, i32 %max, i64 0
 ; CHECK: {{.*}} = shufflevector <2 x i32> %[[VAR]], <2 x i32> poison, <2 x i32> zeroinitializer
-; CHECK: icmp sgt <2 x i32>
-; CHECK: select <2 x i1>
+; CHECK: call <2 x i32> @llvm.smax
 ; CHECK: middle.block
 ; CHECK: call i32 @llvm.vector.reduce.smax.v2i32
 
@@ -27,36 +26,7 @@ for.body:
   %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
   %arrayidx = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %indvars.iv
   %0 = load i32, ptr %arrayidx, align 4
-  %cmp3 = icmp sgt i32 %0, %max.red.08
-  %max.red.0 = select i1 %cmp3, i32 %0, i32 %max.red.08
-  %indvars.iv.next = add i64 %indvars.iv, 1
-  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
-  %exitcond = icmp eq i32 %lftr.wideiv, 1024
-  br i1 %exitcond, label %for.end, label %for.body
-
-for.end:
-  ret i32 %max.red.0
-}
-
-; Turn this into a max reduction. The select has its inputs reversed therefore
-; this is a max reduction.
-; CHECK-LABEL: @max_red_inverse_select(
-; CHECK: icmp slt <2 x i32>
-; CHECK: select <2 x i1>
-; CHECK: middle.block
-; CHECK: call i32 @llvm.vector.reduce.smax.v2i32
-
-define i32 @max_red_inverse_select(i32 %max) {
-entry:
-  br label %for.body
-
-for.body:
-  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
-  %arrayidx = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %indvars.iv
-  %0 = load i32, ptr %arrayidx, align 4
-  %cmp3 = icmp slt i32 %max.red.08, %0
-  %max.red.0 = select i1 %cmp3, i32 %0, i32 %max.red.08
+  %max.red.0 = call i32 @llvm.smax.i32(i32 %0, i32 %max.red.08)
   %indvars.iv.next = add i64 %indvars.iv, 1
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp eq i32 %lftr.wideiv, 1024
@@ -68,8 +38,7 @@ for.end:
 
 ; Turn this into a min reduction.
 ; CHECK-LABEL: @min_red(
-; CHECK: icmp slt <2 x i32>
-; CHECK: select <2 x i1>
+; CHECK: call <2 x i32> @llvm.smin
 ; CHECK: middle.block
 ; CHECK: call i32 @llvm.vector.reduce.smin.v2i32
 
@@ -82,36 +51,7 @@ for.body:
   %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
   %arrayidx = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %indvars.iv
   %0 = load i32, ptr %arrayidx, align 4
-  %cmp3 = icmp slt i32 %0, %max.red.08
-  %max.red.0 = select i1 %cmp3, i32 %0, i32 %max.red.08
-  %indvars.iv.next = add i64 %indvars.iv, 1
-  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
-  %exitcond = icmp eq i32 %lftr.wideiv, 1024
-  br i1 %exitcond, label %for.end, label %for.body
-
-for.end:
-  ret i32 %max.red.0
-}
-
-; Turn this into a min reduction. The select has its inputs reversed therefore
-; this is a min reduction.
-; CHECK-LABEL: @min_red_inverse_select(
-; CHECK: icmp sgt <2 x i32>
-; CHECK: select <2 x i1>
-; CHECK: middle.block
-; CHECK: call i32 @llvm.vector.reduce.smin.v2i32
-
-define i32 @min_red_inverse_select(i32 %max) {
-entry:
-  br label %for.body
-
-for.body:
-  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
-  %arrayidx = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %indvars.iv
-  %0 = load i32, ptr %arrayidx, align 4
-  %cmp3 = icmp sgt i32 %max.red.08, %0
-  %max.red.0 = select i1 %cmp3, i32 %0, i32 %max.red.08
+  %max.red.0 = call i32 @llvm.smin.i32(i32 %0, i32 %max.red.08)
   %indvars.iv.next = add i64 %indvars.iv, 1
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp eq i32 %lftr.wideiv, 1024
@@ -125,8 +65,7 @@ for.end:
 
 ; Turn this into a max reduction.
 ; CHECK-LABEL: @umax_red(
-; CHECK: icmp ugt <2 x i32>
-; CHECK: select <2 x i1>
+; CHECK: call <2 x i32> @llvm.umax
 ; CHECK: middle.block
 ; CHECK: call i32 @llvm.vector.reduce.umax.v2i32
 
@@ -139,36 +78,7 @@ for.body:
   %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
   %arrayidx = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %indvars.iv
   %0 = load i32, ptr %arrayidx, align 4
-  %cmp3 = icmp ugt i32 %0, %max.red.08
-  %max.red.0 = select i1 %cmp3, i32 %0, i32 %max.red.08
-  %indvars.iv.next = add i64 %indvars.iv, 1
-  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
-  %exitcond = icmp eq i32 %lftr.wideiv, 1024
-  br i1 %exitcond, label %for.end, label %for.body
-
-for.end:
-  ret i32 %max.red.0
-}
-
-; Turn this into a max reduction. The select has its inputs reversed therefore
-; this is a max reduction.
-; CHECK-LABEL: @umax_red_inverse_select(
-; CHECK: icmp ult <2 x i32>
-; CHECK: select <2 x i1>
-; CHECK: middle.block
-; CHECK: call i32 @llvm.vector.reduce.umax.v2i32
-
-define i32 @umax_red_inverse_select(i32 %max) {
-entry:
-  br label %for.body
-
-for.body:
-  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
-  %arrayidx = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %indvars.iv
-  %0 = load i32, ptr %arrayidx, align 4
-  %cmp3 = icmp ult i32 %max.red.08, %0
-  %max.red.0 = select i1 %cmp3, i32 %0, i32 %max.red.08
+  %max.red.0 = call i32 @llvm.umax.i32(i32 %0, i32 %max.red.08)
   %indvars.iv.next = add i64 %indvars.iv, 1
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp eq i32 %lftr.wideiv, 1024
@@ -180,8 +90,7 @@ for.end:
 
 ; Turn this into a min reduction.
 ; CHECK-LABEL: @umin_red(
-; CHECK: icmp ult <2 x i32>
-; CHECK: select <2 x i1>
+; CHECK: call <2 x i32> @llvm.umin
 ; CHECK: middle.block
 ; CHECK: call i32 @llvm.vector.reduce.umin.v2i32
 
@@ -194,148 +103,7 @@ for.body:
   %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
   %arrayidx = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %indvars.iv
   %0 = load i32, ptr %arrayidx, align 4
-  %cmp3 = icmp ult i32 %0, %max.red.08
-  %max.red.0 = select i1 %cmp3, i32 %0, i32 %max.red.08
-  %indvars.iv.next = add i64 %indvars.iv, 1
-  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
-  %exitcond = icmp eq i32 %lftr.wideiv, 1024
-  br i1 %exitcond, label %for.end, label %for.body
-
-for.end:
-  ret i32 %max.red.0
-}
-
-; Turn this into a min reduction. The select has its inputs reversed therefore
-; this is a min reduction.
-; CHECK-LABEL: @umin_red_inverse_select(
-; CHECK: icmp ugt <2 x i32>
-; CHECK: select <2 x i1>
-; CHECK: middle.block
-; CHECK: call i32 @llvm.vector.reduce.umin.v2i32
-
-define i32 @umin_red_inverse_select(i32 %max) {
-entry:
-  br label %for.body
-
-for.body:
-  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
-  %arrayidx = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %indvars.iv
-  %0 = load i32, ptr %arrayidx, align 4
-  %cmp3 = icmp ugt i32 %max.red.08, %0
-  %max.red.0 = select i1 %cmp3, i32 %0, i32 %max.red.08
-  %indvars.iv.next = add i64 %indvars.iv, 1
-  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
-  %exitcond = icmp eq i32 %lftr.wideiv, 1024
-  br i1 %exitcond, label %for.end, label %for.body
-
-for.end:
-  ret i32 %max.red.0
-}
-
-; SGE -> SLT
-; Turn this into a min reduction (select inputs are reversed).
-; CHECK-LABEL: @sge_min_red(
-; CHECK: icmp sge <2 x i32>
-; CHECK: select <2 x i1>
-; CHECK: middle.block
-; CHECK: call i32 @llvm.vector.reduce.smin.v2i32
-
-define i32 @sge_min_red(i32 %max) {
-entry:
-  br label %for.body
-
-for.body:
-  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
-  %arrayidx = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %indvars.iv
-  %0 = load i32, ptr %arrayidx, align 4
-  %cmp3 = icmp sge i32 %0, %max.red.08
-  %max.red.0 = select i1 %cmp3, i32 %max.red.08, i32 %0
-  %indvars.iv.next = add i64 %indvars.iv, 1
-  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
-  %exitcond = icmp eq i32 %lftr.wideiv, 1024
-  br i1 %exitcond, label %for.end, label %for.body
-
-for.end:
-  ret i32 %max.red.0
-}
-
-; SLE -> SGT
-; Turn this into a max reduction (select inputs are reversed).
-; CHECK-LABEL: @sle_min_red(
-; CHECK: icmp sle <2 x i32>
-; CHECK: select <2 x i1>
-; CHECK: middle.block
-; CHECK: call i32 @llvm.vector.reduce.smax.v2i32
-
-define i32 @sle_min_red(i32 %max) {
-entry:
-  br label %for.body
-
-for.body:
-  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
-  %arrayidx = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %indvars.iv
-  %0 = load i32, ptr %arrayidx, align 4
-  %cmp3 = icmp sle i32 %0, %max.red.08
-  %max.red.0 = select i1 %cmp3, i32 %max.red.08, i32 %0
-  %indvars.iv.next = add i64 %indvars.iv, 1
-  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
-  %exitcond = icmp eq i32 %lftr.wideiv, 1024
-  br i1 %exitcond, label %for.end, label %for.body
-
-for.end:
-  ret i32 %max.red.0
-}
-
-; UGE -> ULT
-; Turn this into a min reduction (select inputs are reversed).
-; CHECK-LABEL: @uge_min_red(
-; CHECK: icmp uge <2 x i32>
-; CHECK: select <2 x i1>
-; CHECK: middle.block
-; CHECK: call i32 @llvm.vector.reduce.umin.v2i32
-
-define i32 @uge_min_red(i32 %max) {
-entry:
-  br label %for.body
-
-for.body:
-  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
-  %arrayidx = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %indvars.iv
-  %0 = load i32, ptr %arrayidx, align 4
-  %cmp3 = icmp uge i32 %0, %max.red.08
-  %max.red.0 = select i1 %cmp3, i32 %max.red.08, i32 %0
-  %indvars.iv.next = add i64 %indvars.iv, 1
-  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
-  %exitcond = icmp eq i32 %lftr.wideiv, 1024
-  br i1 %exitcond, label %for.end, label %for.body
-
-for.end:
-  ret i32 %max.red.0
-}
-
-; ULE -> UGT
-; Turn this into a max reduction (select inputs are reversed).
-; CHECK-LABEL: @ule_min_red(
-; CHECK: icmp ule <2 x i32>
-; CHECK: select <2 x i1>
-; CHECK: middle.block
-; CHECK: call i32 @llvm.vector.reduce.umax.v2i32
-
-define i32 @ule_min_red(i32 %max) {
-entry:
-  br label %for.body
-
-for.body:
-  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
-  %arrayidx = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %indvars.iv
-  %0 = load i32, ptr %arrayidx, align 4
-  %cmp3 = icmp ule i32 %0, %max.red.08
-  %max.red.0 = select i1 %cmp3, i32 %max.red.08, i32 %0
+  %max.red.0 = call i32 @llvm.umin.i32(i32 %0, i32 %max.red.08)
   %indvars.iv.next = add i64 %indvars.iv, 1
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp eq i32 %lftr.wideiv, 1024
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-inloop-min-max.ll b/llvm/test/Transforms/LoopVectorize/reduction-inloop-min-max.ll
index e6c80d34ab3a0..f0825392a148c 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-inloop-min-max.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-inloop-min-max.ll
@@ -10,18 +10,18 @@ define i32 @reduction_smin(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 1000, [[VECTOR_PH]] ], [ [[RDX_MINMAX:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 1000), [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[WIDE_LOAD]])
-; CHECK-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.smin.i32(i32 [[TMP1]], i32 [[VEC_PHI]])
+; CHECK-NEXT:    [[TMP1]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP1]])
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    ret i32 [[RDX_MINMAX]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
 entry:
   br label %for.body
@@ -31,8 +31,7 @@ for.body:
   %result.08 = phi i32 [ %v0, %for.body ], [ 1000, %entry ]
   %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
   %l0 = load i32, ptr %arrayidx, align 4
-  %c0 = icmp slt i32 %result.08, %l0
-  %v0 = select i1 %c0, i32 %result.08, i32 %l0
+  %v0 = call i32 @llvm.smin(i32 %result.08, i32 %l0)
   %indvars.iv.next = add i64 %indvars.iv, 1
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp eq i32 %lftr.wideiv, 256
@@ -52,18 +51,18 @@ define i32 @reduction_smin_select_ops_flipped(ptr nocapture %A, ptr nocapture %B
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 1000, [[VECTOR_PH]] ], [ [[RDX_MINMAX:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 1000), [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[WIDE_LOAD]])
-; CHECK-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.smax.i32(i32 [[TMP1]], i32 [[VEC_PHI]])
+; CHECK-NEXT:    [[TMP1]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[WIDE_LOAD]], <4 x i32> [[VEC_PHI]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP1]])
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    ret i32 [[RDX_MINMAX]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
 entry:
   br label %for.body
@@ -73,8 +72,7 @@ for.body:
   %result.08 = phi i32 [ %v0, %for.body ], [ 1000, %entry ]
   %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
   %l0 = load i32, ptr %arrayidx, align 4
-  %c0 = icmp slt i32 %result.08, %l0
-  %v0 = select i1 %c0, i32 %l0, i32 %result.08
+  %v0 = call i32 @llvm.smax(i32 %l0, i32 %result.08)
   %indvars.iv.next = add i64 %indvars.iv, 1
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp eq i32 %lftr.wideiv, 256
@@ -136,18 +134,18 @@ define i32 @reduction_umax(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 1000, [[VECTOR_PH]] ], [ [[RDX_MINMAX:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 1000), [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[WIDE_LOAD]])
-; CHECK-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.umax.i32(i32 [[TMP1]], i32 [[VEC_PHI]])
+; CHECK-NEXT:    [[TMP1]] = call <4 x i32> @llvm.umax.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP1]])
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    ret i32 [[RDX_MINMAX]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
 entry:
   br label %for.body
@@ -157,8 +155,7 @@ for.body:
   %result.08 = phi i32 [ %v0, %for.body ], [ 1000, %entry ]
   %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
   %l0 = load i32, ptr %arrayidx, align 4
-  %c0 = icmp ugt i32 %result.08, %l0
-  %v0 = select i1 %c0, i32 %result.08, i32 %l0
+  %v0 = call i32 @llvm.umax(i32 %result.08, i32 %l0)
   %indvars.iv.next = add i64 %indvars.iv, 1
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp eq i32 %lftr.wideiv, 256
@@ -178,18 +175,18 @@ define i32 @reduction_umax_select_ops_flipped(ptr nocapture %A, ptr nocapture %B
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 1000, [[VECTOR_PH]] ], [ [[RDX_MINMAX:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 1000), [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[WIDE_LOAD]])
-; CHECK-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.umin.i32(i32 [[TMP1]], i32 [[VEC_PHI]])
+; CHECK-NEXT:    [[TMP1]] = call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[WIDE_LOAD]], <4 x i32> [[VEC_PHI]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[TMP1]])
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    ret i32 [[RDX_MINMAX]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
 entry:
   br label %for.body
@@ -199,8 +196,7 @@ for.body:
   %result.08 = phi i32 [ %v0, %for.body ], [ 1000, %entry ]
   %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
   %l0 = load i32, ptr %arrayidx, align 4
-  %c0 = icmp ugt i32 %result.08, %l0
-  %v0 = select i1 %c0, i32 %l0, i32 %result.08
+  %v0 = call i32 @llvm.umin(i32 %l0, i32 %result.08)
   %indvars.iv.next = add i64 %indvars.iv, 1
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp eq i32 %lftr.wideiv, 256
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll b/llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll
index c0364c6fc5032..4a1ce5a8a3f4c 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll
@@ -570,13 +570,12 @@ define i32 @reduction_min(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 1000, [[VECTOR_PH]] ], [ [[RDX_MINMAX:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 1000), [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult <4 x i16> [[VEC_IND]], splat (i16 257)
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP24:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP0]], <4 x i32> splat (i32 2147483647))
-; CHECK-NEXT:    [[TMP25:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP24]])
-; CHECK-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.smin.i32(i32 [[TMP25]], i32 [[VEC_PHI]])
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP0]], <4 x i32> poison)
+; CHECK-NEXT:    [[TMP3]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_MASKED_LOAD]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i16> [[VEC_IND]], splat (i16 4)
 ; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
@@ -584,6 +583,8 @@ define i32 @reduction_min(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.end:
+; CHECK-NEXT:    [[TMP4:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP3]], <4 x i32> [[VEC_PHI]]
+; CHECK-NEXT:    [[RDX_MINMAX:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP4]])
 ; CHECK-NEXT:    ret i32 [[RDX_MINMAX]]
 ;
 entry:
@@ -594,8 +595,7 @@ for.body:
   %result.08 = phi i32 [ %v0, %for.body ], [ 1000, %entry ]
   %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
   %l0 = load i32, ptr %arrayidx, align 4
-  %c0 = icmp slt i32 %result.08, %l0
-  %v0 = select i1 %c0, i32 %result.08, i32 %l0
+  %v0 = call i32 @llvm.smin(i32 %result.08, i32 %l0)
   %indvars.iv.next = add i64 %indvars.iv, 1
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp eq i32 %lftr.wideiv, 257
@@ -614,13 +614,12 @@ define i32 @reduction_max(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 1000, [[VECTOR_PH]] ], [ [[RDX_MINMAX:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 1000), [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult <4 x i16> [[VEC_IND]], splat (i16 257)
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP24:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP0]], <4 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP25:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP24]])
-; CHECK-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.umax.i32(i32 [[TMP25]], i32 [[VEC_PHI]])
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP0]], <4 x i32> poison)
+; CHECK-NEXT:    [[TMP3]] = call <4 x i32> @llvm.umax.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_MASKED_LOAD]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i16> [[VEC_IND]], splat (i16 4)
 ; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
@@ -628,6 +627,8 @@ define i32 @reduction_max(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.end:
+; CHECK-NEXT:    [[TMP4:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP3]], <4 x i32> [[VEC_PHI]]
+; CHECK-NEXT:    [[RDX_MINMAX:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP4]])
 ; CHECK-NEXT:    ret i32 [[RDX_MINMAX]]
 ;
 entry:
@@ -638,8 +639,7 @@ for.body:
   %result.08 = phi i32 [ %v0, %for.body ], [ 1000, %entry ]
   %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
   %l0 = load i32, ptr %arrayidx, align 4
-  %c0 = icmp ugt i32 %result.08, %l0
-  %v0 = select i1 %c0, i32 %result.08, i32 %l0
+  %v0 = call i32 @llvm.umax(i32 %result.08, i32 %l0)
   %indvars.iv.next = add i64 %indvars.iv, 1
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp eq i32 %lftr.wideiv, 257
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-predselect.ll b/llvm/test/Transforms/LoopVectorize/reduction-predselect.ll
index 9402688464792..ce451ea1a3d8b 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-predselect.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-predselect.ll
@@ -423,8 +423,7 @@ for.body:
   %result.08 = phi i32 [ %v0, %for.body ], [ 1000, %entry ]
   %arrayidx = getelementptr inbounds i32, ptr %A, i32 %indvars.iv
   %l0 = load i32, ptr %arrayidx, align 4
-  %c0 = icmp slt i32 %result.08, %l0
-  %v0 = select i1 %c0, i32 %result.08, i32 %l0
+  %v0 = call i32 @llvm.smin(i32 %result.08, i32 %l0)
   %indvars.iv.next = add i32 %indvars.iv, 1
   %exitcond = icmp eq i32 %indvars.iv.next, 257
   br i1 %exitcond, label %for.end, label %for.body
@@ -467,8 +466,7 @@ for.body:
   %result.08 = phi i32 [ %v0, %for.body ], [ 1000, %entry ]
   %arrayidx = getelementptr inbounds i32, ptr %A, i32 %indvars.iv
   %l0 = load i32, ptr %arrayidx, align 4
-  %c0 = icmp ugt i32 %result.08, %l0
-  %v0 = select i1 %c0, i32 %result.08, i32 %l0
+  %v0 = call i32 @llvm.umax(i32 %result.08, i32 %l0)
   %indvars.iv.next = add i32 %indvars.iv, 1
   %exitcond = icmp eq i32 %indvars.iv.next, 257
   br i1 %exitcond, label %for.end, label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll
index f89c32c03fa29..10e067c1b5121 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
 ; REQUIRES: asserts
 ; RUN: opt < %s -p 'loop-vectorize' -force-vector-interleave=1 -S \
 ; RUN:   -force-vector-width=4 -debug-only=loop-accesses,loop-vectorize,loop-utils 2> %t | FileCheck %s
@@ -29,9 +29,9 @@ target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 ; DEBUG-NEXT: Start: %src End: ((4 * (zext i32 %m to i64) * (zext i32 %n to i64)) + %src)
 
 define void @diff_checks(ptr nocapture noundef writeonly %dst, ptr nocapture noundef readonly %src, i32 noundef %m, i32 noundef %n) {
-; CHECK-LABEL: define void @diff_checks
-; CHECK-SAME: (ptr noundef writeonly captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define void @diff_checks(
+; CHECK-SAME: ptr noundef writeonly captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[ADD5:%.*]] = add nuw i32 [[N]], 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[ADD5]] to i64
@@ -47,26 +47,26 @@ define void @diff_checks(ptr nocapture noundef writeonly %dst, ptr nocapture nou
 ; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[WIDE_N]], [[WIDE_M]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP8]], 2
 ; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP9]]
-; CHECK-NEXT:    br label [[OUTER_LOOP:%.*]]
-; CHECK:       outer.loop:
-; CHECK-NEXT:    [[IV_OUTER:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_OUTER_NEXT:%.*]], [[INNER_EXIT:%.*]] ]
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[IV_OUTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_OUTER_NEXT:%.*]], %[[INNER_EXIT:.*]] ]
 ; CHECK-NEXT:    [[TMP10:%.*]] = mul nsw i64 [[IV_OUTER]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = mul nsw i64 [[IV_OUTER]], [[TMP1]]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_N]], 4
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
-; CHECK:       vector.memcheck:
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; CHECK-NEXT:    [[STRIDE_CHECK:%.*]] = icmp slt i64 [[TMP7]], 0
 ; CHECK-NEXT:    [[TMP12:%.*]] = or i1 [[FOUND_CONFLICT]], [[STRIDE_CHECK]]
-; CHECK-NEXT:    br i1 [[TMP12]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_N]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_N]], [[N_MOD_VF]]
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP13:%.*]] = add nuw nsw i64 [[INDEX]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP13]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP14]], align 4, !alias.scope [[META0:![0-9]+]]
@@ -75,15 +75,15 @@ define void @diff_checks(ptr nocapture noundef writeonly %dst, ptr nocapture nou
 ; CHECK-NEXT:    store <4 x i32> [[WIDE_LOAD]], ptr [[TMP16]], align 4, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_EXIT]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_LOOP]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
-; CHECK:       inner.loop:
-; CHECK-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_INNER_NEXT:%.*]], [[INNER_LOOP]] ]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[INNER_EXIT]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER_LOOP]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_INNER_NEXT:%.*]], %[[INNER_LOOP]] ]
 ; CHECK-NEXT:    [[TMP18:%.*]] = add nuw nsw i64 [[IV_INNER]], [[TMP10]]
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP18]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX_US]], align 4
@@ -92,12 +92,12 @@ define void @diff_checks(ptr nocapture noundef writeonly %dst, ptr nocapture nou
 ; CHECK-NEXT:    store i32 [[TMP19]], ptr [[ARRAYIDX9_US]], align 4
 ; CHECK-NEXT:    [[IV_INNER_NEXT]] = add nuw nsw i64 [[IV_INNER]], 1
 ; CHECK-NEXT:    [[INNER_EXIT_COND:%.*]] = icmp eq i64 [[IV_INNER_NEXT]], [[WIDE_N]]
-; CHECK-NEXT:    br i1 [[INNER_EXIT_COND]], label [[INNER_EXIT]], label [[INNER_LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK:       inner.exit:
+; CHECK-NEXT:    br i1 [[INNER_EXIT_COND]], label %[[INNER_EXIT]], label %[[INNER_LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       [[INNER_EXIT]]:
 ; CHECK-NEXT:    [[IV_OUTER_NEXT]] = add nuw nsw i64 [[IV_OUTER]], 1
 ; CHECK-NEXT:    [[OUTER_EXIT_COND:%.*]] = icmp eq i64 [[IV_OUTER_NEXT]], [[WIDE_M]]
-; CHECK-NEXT:    br i1 [[OUTER_EXIT_COND]], label [[OUTER_EXIT:%.*]], label [[OUTER_LOOP]]
-; CHECK:       outer.exit:
+; CHECK-NEXT:    br i1 [[OUTER_EXIT_COND]], label %[[OUTER_EXIT:.*]], label %[[OUTER_LOOP]]
+; CHECK:       [[OUTER_EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -158,9 +158,9 @@ outer.exit:
 ; DEBUG-NEXT: Start: %src End: ((4 * (zext i32 %m to i64) * (zext i32 %n to i64)) + %src)
 
 define void @full_checks(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, i32 noundef %m, i32 noundef %n) {
-; CHECK-LABEL: define void @full_checks
-; CHECK-SAME: (ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define void @full_checks(
+; CHECK-SAME: ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; CHECK-NEXT:    [[WIDE_M:%.*]] = zext i32 [[M]] to i64
 ; CHECK-NEXT:    [[WIDE_N:%.*]] = zext i32 [[N]] to i64
@@ -168,23 +168,23 @@ define void @full_checks(ptr nocapture noundef %dst, ptr nocapture noundef reado
 ; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 2
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP2]]
-; CHECK-NEXT:    br label [[OUTER_LOOP:%.*]]
-; CHECK:       outer.loop:
-; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[OUTER_IV_NEXT:%.*]], [[INNER_EXIT:%.*]] ]
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[INNER_EXIT:.*]] ]
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nsw i64 [[OUTER_IV]], [[TMP0]]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_N]], 4
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
-; CHECK:       vector.memcheck:
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_N]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_N]], [[N_MOD_VF]]
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP4:%.*]] = add nuw nsw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP4]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4, !alias.scope [[META9:![0-9]+]]
@@ -194,15 +194,15 @@ define void @full_checks(ptr nocapture noundef %dst, ptr nocapture noundef reado
 ; CHECK-NEXT:    store <4 x i32> [[TMP7]], ptr [[TMP6]], align 4, !alias.scope [[META12]], !noalias [[META9]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
-; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_EXIT]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_LOOP]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
-; CHECK:       inner.loop:
-; CHECK-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_INNER_NEXT:%.*]], [[INNER_LOOP]] ]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[INNER_EXIT]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER_LOOP]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_INNER_NEXT:%.*]], %[[INNER_LOOP]] ]
 ; CHECK-NEXT:    [[TMP9:%.*]] = add nuw nsw i64 [[IV_INNER]], [[TMP3]]
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP9]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX_US]], align 4
@@ -212,12 +212,12 @@ define void @full_checks(ptr nocapture noundef %dst, ptr nocapture noundef reado
 ; CHECK-NEXT:    store i32 [[ADD9_US]], ptr [[ARRAYIDX8_US]], align 4
 ; CHECK-NEXT:    [[IV_INNER_NEXT]] = add nuw nsw i64 [[IV_INNER]], 1
 ; CHECK-NEXT:    [[INNER_EXIT_COND:%.*]] = icmp eq i64 [[IV_INNER_NEXT]], [[WIDE_N]]
-; CHECK-NEXT:    br i1 [[INNER_EXIT_COND]], label [[INNER_EXIT]], label [[INNER_LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
-; CHECK:       inner.exit:
+; CHECK-NEXT:    br i1 [[INNER_EXIT_COND]], label %[[INNER_EXIT]], label %[[INNER_LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK:       [[INNER_EXIT]]:
 ; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
 ; CHECK-NEXT:    [[OUTER_EXIT_COND:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[WIDE_M]]
-; CHECK-NEXT:    br i1 [[OUTER_EXIT_COND]], label [[OUTER_EXIT:%.*]], label [[OUTER_LOOP]]
-; CHECK:       outer.exit:
+; CHECK-NEXT:    br i1 [[OUTER_EXIT_COND]], label %[[OUTER_EXIT:.*]], label %[[OUTER_LOOP]]
+; CHECK:       [[OUTER_EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -279,9 +279,9 @@ outer.exit:
 ; DEBUG-NEXT: Start: %src End: ((4 * (zext i32 %m to i64) * (zext i32 %n to i64)) + %src)
 
 define void @full_checks_diff_strides(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, i32 noundef %m, i32 noundef %n) {
-; CHECK-LABEL: define void @full_checks_diff_strides
-; CHECK-SAME: (ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define void @full_checks_diff_strides(
+; CHECK-SAME: ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[WIDE_M:%.*]] = zext i32 [[M]] to i64
 ; CHECK-NEXT:    [[WIDE_N:%.*]] = zext i32 [[N]] to i64
 ; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i64 [[WIDE_M]], -1
@@ -293,26 +293,26 @@ define void @full_checks_diff_strides(ptr nocapture noundef %dst, ptr nocapture
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[WIDE_N]], [[WIDE_M]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP6]]
-; CHECK-NEXT:    br label [[OUTER_LOOP:%.*]]
-; CHECK:       outer.loop:
-; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[OUTER_IV_NEXT:%.*]], [[INNER_EXIT:%.*]] ]
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[INNER_EXIT:.*]] ]
 ; CHECK-NEXT:    [[NPLUS1:%.*]] = add nuw nsw i32 [[N]], 1
 ; CHECK-NEXT:    [[WIDE_NPLUS1:%.*]] = zext i32 [[NPLUS1]] to i64
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul nsw i64 [[OUTER_IV]], [[WIDE_N]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = mul nsw i64 [[OUTER_IV]], [[WIDE_NPLUS1]]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_N]], 4
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
-; CHECK:       vector.memcheck:
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_N]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_N]], [[N_MOD_VF]]
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP9:%.*]] = add nuw nsw i64 [[INDEX]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP9]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4, !alias.scope [[META16:![0-9]+]]
@@ -323,15 +323,15 @@ define void @full_checks_diff_strides(ptr nocapture noundef %dst, ptr nocapture
 ; CHECK-NEXT:    store <4 x i32> [[TMP13]], ptr [[TMP12]], align 4, !alias.scope [[META19]], !noalias [[META16]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
-; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_EXIT]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_LOOP]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
-; CHECK:       inner.loop:
-; CHECK-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_INNER_NEXT:%.*]], [[INNER_LOOP]] ]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[INNER_EXIT]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER_LOOP]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_INNER_NEXT:%.*]], %[[INNER_LOOP]] ]
 ; CHECK-NEXT:    [[TMP15:%.*]] = add nuw nsw i64 [[IV_INNER]], [[TMP7]]
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP15]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX_US]], align 4
@@ -342,12 +342,12 @@ define void @full_checks_diff_strides(ptr nocapture noundef %dst, ptr nocapture
 ; CHECK-NEXT:    store i32 [[ADD9_US]], ptr [[ARRAYIDX8_US]], align 4
 ; CHECK-NEXT:    [[IV_INNER_NEXT]] = add nuw nsw i64 [[IV_INNER]], 1
 ; CHECK-NEXT:    [[INNER_EXIT_COND:%.*]] = icmp eq i64 [[IV_INNER_NEXT]], [[WIDE_N]]
-; CHECK-NEXT:    br i1 [[INNER_EXIT_COND]], label [[INNER_EXIT]], label [[INNER_LOOP]], !llvm.loop [[LOOP22:![0-9]+]]
-; CHECK:       inner.exit:
+; CHECK-NEXT:    br i1 [[INNER_EXIT_COND]], label %[[INNER_EXIT]], label %[[INNER_LOOP]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK:       [[INNER_EXIT]]:
 ; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
 ; CHECK-NEXT:    [[OUTER_EXIT_COND:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[WIDE_M]]
-; CHECK-NEXT:    br i1 [[OUTER_EXIT_COND]], label [[OUTER_EXIT:%.*]], label [[OUTER_LOOP]]
-; CHECK:       outer.exit:
+; CHECK-NEXT:    br i1 [[OUTER_EXIT_COND]], label %[[OUTER_EXIT:.*]], label %[[OUTER_LOOP]]
+; CHECK:       [[OUTER_EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -401,9 +401,9 @@ outer.exit:
 ; DEBUG-NOT: LAA: Expanded RT check for range to include outer loop in order to permit hoisting
 
 define void @diff_checks_src_start_invariant(ptr nocapture noundef writeonly %dst, ptr nocapture noundef readonly %src, i32 noundef %m, i32 noundef %n) {
-; CHECK-LABEL: define void @diff_checks_src_start_invariant
-; CHECK-SAME: (ptr noundef writeonly captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define void @diff_checks_src_start_invariant(
+; CHECK-SAME: ptr noundef writeonly captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[SRC2:%.*]] = ptrtoaddr ptr [[SRC]] to i64
 ; CHECK-NEXT:    [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64
 ; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
@@ -411,23 +411,23 @@ define void @diff_checks_src_start_invariant(ptr nocapture noundef writeonly %ds
 ; CHECK-NEXT:    [[WIDE_N:%.*]] = zext i32 [[N]] to i64
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[DST1]], [[SRC2]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[WIDE_N]], 2
-; CHECK-NEXT:    br label [[OUTER_LOOP:%.*]]
-; CHECK:       outer.loop:
-; CHECK-NEXT:    [[IV_OUTER:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_OUTER_NEXT:%.*]], [[INNER_LOOP_EXIT:%.*]] ]
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[IV_OUTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_OUTER_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ]
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], [[IV_OUTER]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul nsw i64 [[IV_OUTER]], [[TMP0]]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_N]], 4
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
-; CHECK:       vector.memcheck:
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], 16
-; CHECK-NEXT:    br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
+; CHECK-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_N]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_N]], [[N_MOD_VF]]
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = add nuw nsw i64 [[INDEX]], [[TMP5]]
@@ -435,15 +435,15 @@ define void @diff_checks_src_start_invariant(ptr nocapture noundef writeonly %ds
 ; CHECK-NEXT:    store <4 x i32> [[WIDE_LOAD]], ptr [[TMP8]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
-; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_LOOP_EXIT]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_LOOP]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
-; CHECK:       inner.loop:
-; CHECK-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_INNER_NEXT:%.*]], [[INNER_LOOP]] ]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[INNER_LOOP_EXIT]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER_LOOP]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_INNER_NEXT:%.*]], %[[INNER_LOOP]] ]
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV_INNER]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX_US]], align 4
 ; CHECK-NEXT:    [[TMP11:%.*]] = add nuw nsw i64 [[IV_INNER]], [[TMP5]]
@@ -451,12 +451,12 @@ define void @diff_checks_src_start_invariant(ptr nocapture noundef writeonly %ds
 ; CHECK-NEXT:    store i32 [[TMP10]], ptr [[ARRAYIDX6_US]], align 4
 ; CHECK-NEXT:    [[IV_INNER_NEXT]] = add nuw nsw i64 [[IV_INNER]], 1
 ; CHECK-NEXT:    [[INNER_EXIT_COND:%.*]] = icmp eq i64 [[IV_INNER_NEXT]], [[WIDE_N]]
-; CHECK-NEXT:    br i1 [[INNER_EXIT_COND]], label [[INNER_LOOP_EXIT]], label [[INNER_LOOP]], !llvm.loop [[LOOP24:![0-9]+]]
-; CHECK:       inner.loop.exit:
+; CHECK-NEXT:    br i1 [[INNER_EXIT_COND]], label %[[INNER_LOOP_EXIT]], label %[[INNER_LOOP]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK:       [[INNER_LOOP_EXIT]]:
 ; CHECK-NEXT:    [[IV_OUTER_NEXT]] = add nuw nsw i64 [[IV_OUTER]], 1
 ; CHECK-NEXT:    [[OUTER_EXIT_COND:%.*]] = icmp eq i64 [[IV_OUTER_NEXT]], [[WIDE_M]]
-; CHECK-NEXT:    br i1 [[OUTER_EXIT_COND]], label [[OUTER_LOOP_EXIT:%.*]], label [[OUTER_LOOP]]
-; CHECK:       outer.loop.exit:
+; CHECK-NEXT:    br i1 [[OUTER_EXIT_COND]], label %[[OUTER_LOOP_EXIT:.*]], label %[[OUTER_LOOP]]
+; CHECK:       [[OUTER_LOOP_EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -509,9 +509,9 @@ outer.loop.exit:
 ; DEBUG-NEXT: Start: %src End: ((4 * (zext i32 %n to i64))<nuw><nsw> + %src)
 
 define void @full_checks_src_start_invariant(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, i32 noundef %m, i32 noundef %n) {
-; CHECK-LABEL: define void @full_checks_src_start_invariant
-; CHECK-SAME: (ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define void @full_checks_src_start_invariant(
+; CHECK-SAME: ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; CHECK-NEXT:    [[WIDE_M:%.*]] = zext i32 [[M]] to i64
 ; CHECK-NEXT:    [[WIDE_N:%.*]] = zext i32 [[N]] to i64
@@ -520,23 +520,23 @@ define void @full_checks_src_start_invariant(ptr nocapture noundef %dst, ptr noc
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i64 [[WIDE_N]], 2
 ; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]]
-; CHECK-NEXT:    br label [[OUTER_LOOP:%.*]]
-; CHECK:       outer.loop:
-; CHECK-NEXT:    [[IV_OUTER:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_OUTER_NEXT:%.*]], [[INNER_LOOP_EXIT:%.*]] ]
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[IV_OUTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_OUTER_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ]
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul nsw i64 [[IV_OUTER]], [[TMP0]]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_N]], 4
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
-; CHECK:       vector.memcheck:
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_N]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_N]], [[N_MOD_VF]]
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4, !alias.scope [[META25:![0-9]+]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = add nuw nsw i64 [[INDEX]], [[TMP4]]
@@ -546,15 +546,15 @@ define void @full_checks_src_start_invariant(ptr nocapture noundef %dst, ptr noc
 ; CHECK-NEXT:    store <4 x i32> [[TMP8]], ptr [[TMP7]], align 4, !alias.scope [[META28]], !noalias [[META25]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
-; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_LOOP_EXIT]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_LOOP]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
-; CHECK:       inner.loop:
-; CHECK-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_INNER_NEXT:%.*]], [[INNER_LOOP]] ]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[INNER_LOOP_EXIT]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER_LOOP]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_INNER_NEXT:%.*]], %[[INNER_LOOP]] ]
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV_INNER]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX_US]], align 4
 ; CHECK-NEXT:    [[TMP11:%.*]] = add nuw nsw i64 [[IV_INNER]], [[TMP4]]
@@ -564,12 +564,12 @@ define void @full_checks_src_start_invariant(ptr nocapture noundef %dst, ptr noc
 ; CHECK-NEXT:    store i32 [[ADD7_US]], ptr [[ARRAYIDX6_US]], align 4
 ; CHECK-NEXT:    [[IV_INNER_NEXT]] = add nuw nsw i64 [[IV_INNER]], 1
 ; CHECK-NEXT:    [[INNER_EXIT_COND:%.*]] = icmp eq i64 [[IV_INNER_NEXT]], [[WIDE_N]]
-; CHECK-NEXT:    br i1 [[INNER_EXIT_COND]], label [[INNER_LOOP_EXIT]], label [[INNER_LOOP]], !llvm.loop [[LOOP31:![0-9]+]]
-; CHECK:       inner.loop.exit:
+; CHECK-NEXT:    br i1 [[INNER_EXIT_COND]], label %[[INNER_LOOP_EXIT]], label %[[INNER_LOOP]], !llvm.loop [[LOOP31:![0-9]+]]
+; CHECK:       [[INNER_LOOP_EXIT]]:
 ; CHECK-NEXT:    [[IV_OUTER_NEXT]] = add nuw nsw i64 [[IV_OUTER]], 1
 ; CHECK-NEXT:    [[OUTER_EXIT_COND:%.*]] = icmp eq i64 [[IV_OUTER_NEXT]], [[WIDE_M]]
-; CHECK-NEXT:    br i1 [[OUTER_EXIT_COND]], label [[OUTER_LOOP_EXIT:%.*]], label [[OUTER_LOOP]]
-; CHECK:       outer.loop.exit:
+; CHECK-NEXT:    br i1 [[OUTER_EXIT_COND]], label %[[OUTER_LOOP_EXIT:.*]], label %[[OUTER_LOOP]]
+; CHECK:       [[OUTER_LOOP_EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -629,9 +629,9 @@ outer.loop.exit:
 ; DEBUG-NEXT: Start: {%src,+,(4 * (zext i32 %n to i64) * (zext i32 %o to i64))}<%outer.outer.loop> End: {((4 * (zext i32 %o to i64))<nuw><nsw> + %src),+,(4 * (zext i32 %n to i64) * (zext i32 %o to i64))}<%outer.outer.loop>
 
 define void @triple_nested_loop_mixed_access(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, i32 noundef %m, i32 noundef %n, i32 noundef %o) {
-; CHECK-LABEL: define void @triple_nested_loop_mixed_access
-; CHECK-SAME: (ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]], i32 noundef [[O:%.*]]) {
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define void @triple_nested_loop_mixed_access(
+; CHECK-SAME: ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]], i32 noundef [[O:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[O]], 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[O]] to i64
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[N]] to i64
@@ -646,9 +646,9 @@ define void @triple_nested_loop_mixed_access(ptr nocapture noundef %dst, ptr noc
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = shl i64 [[TMP7]], 2
 ; CHECK-NEXT:    [[TMP9:%.*]] = shl nuw nsw i64 [[WIDE_TRIP_COUNT]], 2
-; CHECK-NEXT:    br label [[OUTER_OUTER_LOOP:%.*]]
-; CHECK:       outer.outer.loop:
-; CHECK-NEXT:    [[OUTER_OUTER_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[OUTER_OUTER_IV_NEXT:%.*]], [[OUTER_LOOP_END:%.*]] ]
+; CHECK-NEXT:    br label %[[OUTER_OUTER_LOOP:.*]]
+; CHECK:       [[OUTER_OUTER_LOOP]]:
+; CHECK-NEXT:    [[OUTER_OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_OUTER_IV_NEXT:%.*]], %[[OUTER_LOOP_END:.*]] ]
 ; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP4]], [[OUTER_OUTER_IV]]
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP10]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP6]], [[TMP10]]
@@ -660,24 +660,24 @@ define void @triple_nested_loop_mixed_access(ptr nocapture noundef %dst, ptr noc
 ; CHECK-NEXT:    [[TMP14:%.*]] = mul nsw i64 [[OUTER_OUTER_IV]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = mul nsw i64 [[TMP14]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul nsw i64 [[TMP14]], [[TMP2]]
-; CHECK-NEXT:    br label [[OUTER_LOOP:%.*]]
-; CHECK:       outer.loop:
-; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ [[OUTER_IV_NEXT:%.*]], [[INNER_LOOP_END:%.*]] ], [ 0, [[OUTER_OUTER_LOOP]] ]
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ [[OUTER_IV_NEXT:%.*]], %[[INNER_LOOP_END:.*]] ], [ 0, %[[OUTER_OUTER_LOOP]] ]
 ; CHECK-NEXT:    [[TMP17:%.*]] = mul nsw i64 [[OUTER_IV]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = add nuw nsw i64 [[TMP17]], [[TMP16]]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
-; CHECK:       vector.memcheck:
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP3]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP1]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP19:%.*]] = add nuw nsw i64 [[INDEX]], [[TMP15]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP19]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP20]], align 4, !alias.scope [[META32:![0-9]+]]
@@ -688,15 +688,15 @@ define void @triple_nested_loop_mixed_access(ptr nocapture noundef %dst, ptr noc
 ; CHECK-NEXT:    store <4 x i32> [[TMP23]], ptr [[TMP22]], align 4, !alias.scope [[META35]], !noalias [[META32]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
-; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_LOOP_END]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_LOOP]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
-; CHECK:       inner.loop:
-; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ [[INNER_IV_NEXT:%.*]], [[INNER_LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[INNER_LOOP_END]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER_LOOP]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ [[INNER_IV_NEXT:%.*]], %[[INNER_LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[TMP25:%.*]] = add nuw nsw i64 [[INNER_IV]], [[TMP15]]
 ; CHECK-NEXT:    [[ARRAYIDX_US_US_US:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP25]]
 ; CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[ARRAYIDX_US_US_US]], align 4
@@ -707,16 +707,16 @@ define void @triple_nested_loop_mixed_access(ptr nocapture noundef %dst, ptr noc
 ; CHECK-NEXT:    store i32 [[ADD18_US_US_US]], ptr [[ARRAYIDX17_US_US_US]], align 4
 ; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[INNER_LOOP_END]], label [[INNER_LOOP]], !llvm.loop [[LOOP38:![0-9]+]]
-; CHECK:       inner.loop.end:
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[INNER_LOOP_END]], label %[[INNER_LOOP]], !llvm.loop [[LOOP38:![0-9]+]]
+; CHECK:       [[INNER_LOOP_END]]:
 ; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
 ; CHECK-NEXT:    [[EXIT_OUTER:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[WIDE_TRIP_COUNT60]]
-; CHECK-NEXT:    br i1 [[EXIT_OUTER]], label [[OUTER_LOOP_END]], label [[OUTER_LOOP]]
-; CHECK:       outer.loop.end:
+; CHECK-NEXT:    br i1 [[EXIT_OUTER]], label %[[OUTER_LOOP_END]], label %[[OUTER_LOOP]]
+; CHECK:       [[OUTER_LOOP_END]]:
 ; CHECK-NEXT:    [[OUTER_OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_OUTER_IV]], 1
 ; CHECK-NEXT:    [[EXIT_OUTER_OUTER:%.*]] = icmp eq i64 [[OUTER_OUTER_IV_NEXT]], [[WIDE_TRIP_COUNT68]]
-; CHECK-NEXT:    br i1 [[EXIT_OUTER_OUTER]], label [[EXIT:%.*]], label [[OUTER_OUTER_LOOP]]
-; CHECK:       exit:
+; CHECK-NEXT:    br i1 [[EXIT_OUTER_OUTER]], label %[[EXIT:.*]], label %[[OUTER_OUTER_LOOP]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -791,15 +791,15 @@ exit:
 ; DEBUG-NEXT: Start: {%src,+,(4 * (zext i32 %n to i64))<nuw><nsw>}<%outer.loop> End: {((4 * (zext i32 %n to i64))<nuw><nsw> + %src),+,(4 * (zext i32 %n to i64))<nuw><nsw>}<%outer.loop>
 
 define void @uncomputable_outer_tc(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, ptr nocapture noundef readonly %str, i32 noundef %n) {
-; CHECK-LABEL: define void @uncomputable_outer_tc
-; CHECK-SAME: (ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], ptr noundef readonly captures(none) [[STR:%.*]], i32 noundef [[N:%.*]]) {
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define void @uncomputable_outer_tc(
+; CHECK-SAME: ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], ptr noundef readonly captures(none) [[STR:%.*]], i32 noundef [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[STR]], align 1
 ; CHECK-NEXT:    [[CMP_NOT23:%.*]] = icmp ne i8 [[TMP0]], 0
 ; CHECK-NEXT:    [[CMP221:%.*]] = icmp sgt i32 [[N]], 0
 ; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[CMP_NOT23]], [[CMP221]]
-; CHECK-NEXT:    br i1 [[OR_COND]], label [[OUTER_LOOP_PREHEADER:%.*]], label [[WHILE_END:%.*]]
-; CHECK:       outer.loop.preheader:
+; CHECK-NEXT:    br i1 [[OR_COND]], label %[[OUTER_LOOP_PREHEADER:.*]], label %[[WHILE_END:.*]]
+; CHECK:       [[OUTER_LOOP_PREHEADER]]:
 ; CHECK-NEXT:    [[ADD6:%.*]] = add nuw nsw i32 [[N]], 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[N]] to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[ADD6]] to i64
@@ -807,9 +807,9 @@ define void @uncomputable_outer_tc(ptr nocapture noundef %dst, ptr nocapture nou
 ; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[WIDE_TRIP_COUNT]], 2
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[WIDE_TRIP_COUNT]], 2
-; CHECK-NEXT:    br label [[OUTER_LOOP:%.*]]
-; CHECK:       outer.loop:
-; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, [[OUTER_LOOP_PREHEADER]] ], [ [[OUTER_IV_NEXT:%.*]], [[INNER_LOOP_EXIT:%.*]] ]
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[OUTER_LOOP_PREHEADER]] ], [ [[OUTER_IV_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ]
 ; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP3]], [[OUTER_IV]]
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP6]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP4]], [[TMP6]]
@@ -821,18 +821,18 @@ define void @uncomputable_outer_tc(ptr nocapture noundef %dst, ptr nocapture nou
 ; CHECK-NEXT:    [[TMP10:%.*]] = mul nsw i64 [[OUTER_IV]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = mul nsw i64 [[OUTER_IV]], [[TMP2]]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
-; CHECK:       vector.memcheck:
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP3]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP1]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP12:%.*]] = add nsw i64 [[INDEX]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP12]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP13]], align 4, !alias.scope [[META39:![0-9]+]]
@@ -843,15 +843,15 @@ define void @uncomputable_outer_tc(ptr nocapture noundef %dst, ptr nocapture nou
 ; CHECK-NEXT:    store <4 x i32> [[TMP16]], ptr [[TMP15]], align 4, !alias.scope [[META42]], !noalias [[META39]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
-; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_LOOP_EXIT]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_LOOP]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
-; CHECK:       inner.loop:
-; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INNER_IV_NEXT:%.*]], [[INNER_LOOP]] ]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[INNER_LOOP_EXIT]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER_LOOP]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_LOOP]] ]
 ; CHECK-NEXT:    [[TMP18:%.*]] = add nsw i64 [[INNER_IV]], [[TMP10]]
 ; CHECK-NEXT:    [[ARRAYIDX5_US:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP18]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX5_US]], align 4
@@ -862,16 +862,16 @@ define void @uncomputable_outer_tc(ptr nocapture noundef %dst, ptr nocapture nou
 ; CHECK-NEXT:    store i32 [[ADD11_US]], ptr [[ARRAYIDX10_US]], align 4
 ; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[INNER_LOOP_EXIT]], label [[INNER_LOOP]], !llvm.loop [[LOOP45:![0-9]+]]
-; CHECK:       inner.loop.exit:
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[INNER_LOOP_EXIT]], label %[[INNER_LOOP]], !llvm.loop [[LOOP45:![0-9]+]]
+; CHECK:       [[INNER_LOOP_EXIT]]:
 ; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add i64 [[OUTER_IV]], 1
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i8, ptr [[STR]], i64 [[OUTER_IV_NEXT]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = load i8, ptr [[ARRAYIDX_US]], align 1
 ; CHECK-NEXT:    [[CMP_NOT_US:%.*]] = icmp eq i8 [[TMP22]], 0
-; CHECK-NEXT:    br i1 [[CMP_NOT_US]], label [[WHILE_END_LOOPEXIT:%.*]], label [[OUTER_LOOP]]
-; CHECK:       while.end.loopexit:
-; CHECK-NEXT:    br label [[WHILE_END]]
-; CHECK:       while.end:
+; CHECK-NEXT:    br i1 [[CMP_NOT_US]], label %[[WHILE_END_LOOPEXIT:.*]], label %[[OUTER_LOOP]]
+; CHECK:       [[WHILE_END_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[WHILE_END]]
+; CHECK:       [[WHILE_END]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -943,14 +943,14 @@ while.end:
 ; DEBUG-NEXT: Start: %src End: (4 + (4 * (zext i32 %n to i64))<nuw><nsw> + (4 * (sext i32 %stride2 to i64) * (-1 + (zext i32 %m to i64))<nsw>) + %src)
 
 define void @decreasing_inner_iv(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, i32 noundef %stride1, i32 noundef %stride2, i32 noundef %m, i32 noundef %n) {
-; CHECK-LABEL: define void @decreasing_inner_iv
-; CHECK-SAME: (ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[STRIDE1:%.*]], i32 noundef [[STRIDE2:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define void @decreasing_inner_iv(
+; CHECK-SAME: ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[STRIDE1:%.*]], i32 noundef [[STRIDE2:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[CMP20:%.*]] = icmp sgt i32 [[M]], 0
 ; CHECK-NEXT:    [[CMP218:%.*]] = icmp sgt i32 [[N]], -1
 ; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[CMP20]], [[CMP218]]
-; CHECK-NEXT:    br i1 [[OR_COND]], label [[OUTER_LOOP_PRE:%.*]], label [[EXIT:%.*]]
-; CHECK:       outer.loop.pre:
+; CHECK-NEXT:    br i1 [[OR_COND]], label %[[OUTER_LOOP_PRE:.*]], label %[[EXIT:.*]]
+; CHECK:       [[OUTER_LOOP_PRE]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[STRIDE2]] to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[STRIDE1]] to i64
@@ -969,15 +969,15 @@ define void @decreasing_inner_iv(ptr nocapture noundef %dst, ptr nocapture nound
 ; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP12]], 4
 ; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP13]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = shl nsw i64 [[TMP1]], 2
-; CHECK-NEXT:    br label [[OUTER_LOOP:%.*]]
-; CHECK:       outer.loop:
-; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, [[OUTER_LOOP_PRE]] ], [ [[OUTER_IV_NEXT:%.*]], [[INNER_LOOP_EXIT:%.*]] ]
-; CHECK-NEXT:    [[TMP16:%.*]] = mul nsw i64 [[OUTER_IV]], [[TMP1]]
-; CHECK-NEXT:    [[TMP17:%.*]] = mul nsw i64 [[OUTER_IV]], [[TMP2]]
-; CHECK-NEXT:    [[TMP15:%.*]] = add nuw nsw i64 [[TMP0]], 1
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP15]], 4
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
-; CHECK:       vector.memcheck:
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[OUTER_LOOP_PRE]] ], [ [[OUTER_IV_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ]
+; CHECK-NEXT:    [[TMP15:%.*]] = mul nsw i64 [[OUTER_IV]], [[TMP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = mul nsw i64 [[OUTER_IV]], [[TMP2]]
+; CHECK-NEXT:    [[TMP17:%.*]] = add nuw nsw i64 [[TMP0]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP17]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
@@ -985,57 +985,57 @@ define void @decreasing_inner_iv(ptr nocapture noundef %dst, ptr nocapture nound
 ; CHECK-NEXT:    [[TMP18:%.*]] = or i1 [[FOUND_CONFLICT]], [[STRIDE_CHECK]]
 ; CHECK-NEXT:    [[STRIDE_CHECK2:%.*]] = icmp slt i64 [[TMP14]], 0
 ; CHECK-NEXT:    [[TMP19:%.*]] = or i1 [[TMP18]], [[STRIDE_CHECK2]]
-; CHECK-NEXT:    br i1 [[TMP19]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP15]], 4
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP15]], [[N_MOD_VF]]
+; CHECK-NEXT:    br i1 [[TMP19]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP17]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP17]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = sub i64 [[TMP0]], [[N_VEC]]
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 [[TMP0]], [[INDEX]]
-; CHECK-NEXT:    [[TMP21:%.*]] = add nsw i64 [[OFFSET_IDX]], [[TMP16]]
-; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP21]]
-; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[TMP22]], i64 -3
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP23]], align 4, !alias.scope [[META46:![0-9]+]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP21:%.*]] = sub i64 [[TMP0]], [[INDEX]]
+; CHECK-NEXT:    [[TMP22:%.*]] = add nsw i64 [[TMP21]], [[TMP15]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[TMP23]], i64 -3
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP24]], align 4, !alias.scope [[META46:![0-9]+]]
 ; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[TMP24:%.*]] = add nsw i64 [[OFFSET_IDX]], [[TMP17]]
-; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP24]]
-; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 -3
-; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP26]], align 4, !alias.scope [[META49:![0-9]+]], !noalias [[META46]]
+; CHECK-NEXT:    [[TMP25:%.*]] = add nsw i64 [[TMP21]], [[TMP16]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[TMP26]], i64 -3
+; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP27]], align 4, !alias.scope [[META49:![0-9]+]], !noalias [[META46]]
 ; CHECK-NEXT:    [[REVERSE4:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD3]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[TMP27:%.*]] = add nsw <4 x i32> [[REVERSE4]], [[REVERSE]]
-; CHECK-NEXT:    [[REVERSE5:%.*]] = shufflevector <4 x i32> [[TMP27]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    store <4 x i32> [[REVERSE5]], ptr [[TMP26]], align 4, !alias.scope [[META49]], !noalias [[META46]]
+; CHECK-NEXT:    [[TMP28:%.*]] = add nsw <4 x i32> [[REVERSE4]], [[REVERSE]]
+; CHECK-NEXT:    [[REVERSE5:%.*]] = shufflevector <4 x i32> [[TMP28]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    store <4 x i32> [[REVERSE5]], ptr [[TMP27]], align 4, !alias.scope [[META49]], !noalias [[META46]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP51:![0-9]+]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP15]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_LOOP_EXIT]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP20]], [[MIDDLE_BLOCK]] ], [ [[TMP0]], [[OUTER_LOOP]] ], [ [[TMP0]], [[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
-; CHECK:       inner.loop:
-; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INNER_IV_NEXT:%.*]], [[INNER_LOOP]] ]
-; CHECK-NEXT:    [[TMP29:%.*]] = add nsw i64 [[INNER_IV]], [[TMP16]]
-; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP29]]
-; CHECK-NEXT:    [[TMP30:%.*]] = load i32, ptr [[ARRAYIDX_US]], align 4
-; CHECK-NEXT:    [[TMP31:%.*]] = add nsw i64 [[INNER_IV]], [[TMP17]]
-; CHECK-NEXT:    [[ARRAYIDX8_US:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP31]]
-; CHECK-NEXT:    [[TMP32:%.*]] = load i32, ptr [[ARRAYIDX8_US]], align 4
-; CHECK-NEXT:    [[ADD9_US:%.*]] = add nsw i32 [[TMP32]], [[TMP30]]
+; CHECK-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP51:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP17]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[INNER_LOOP_EXIT]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP20]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[OUTER_LOOP]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_LOOP]] ]
+; CHECK-NEXT:    [[TMP30:%.*]] = add nsw i64 [[INNER_IV]], [[TMP15]]
+; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP30]]
+; CHECK-NEXT:    [[TMP31:%.*]] = load i32, ptr [[ARRAYIDX_US]], align 4
+; CHECK-NEXT:    [[TMP32:%.*]] = add nsw i64 [[INNER_IV]], [[TMP16]]
+; CHECK-NEXT:    [[ARRAYIDX8_US:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP32]]
+; CHECK-NEXT:    [[TMP33:%.*]] = load i32, ptr [[ARRAYIDX8_US]], align 4
+; CHECK-NEXT:    [[ADD9_US:%.*]] = add nsw i32 [[TMP33]], [[TMP31]]
 ; CHECK-NEXT:    store i32 [[ADD9_US]], ptr [[ARRAYIDX8_US]], align 4
 ; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nsw i64 [[INNER_IV]], -1
 ; CHECK-NEXT:    [[CMP2_US:%.*]] = icmp sgt i64 [[INNER_IV]], 0
-; CHECK-NEXT:    br i1 [[CMP2_US]], label [[INNER_LOOP]], label [[INNER_LOOP_EXIT]], !llvm.loop [[LOOP52:![0-9]+]]
-; CHECK:       inner.loop.exit:
+; CHECK-NEXT:    br i1 [[CMP2_US]], label %[[INNER_LOOP]], label %[[INNER_LOOP_EXIT]], !llvm.loop [[LOOP52:![0-9]+]]
+; CHECK:       [[INNER_LOOP_EXIT]]:
 ; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[OUTER_LOOP_EXIT:%.*]], label [[OUTER_LOOP]]
-; CHECK:       outer.loop.exit:
-; CHECK-NEXT:    br label [[EXIT]]
-; CHECK:       exit:
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[OUTER_LOOP_EXIT:.*]], label %[[OUTER_LOOP]]
+; CHECK:       [[OUTER_LOOP_EXIT]]:
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -1107,14 +1107,14 @@ exit:
 ; DEBUG-NEXT: Start: ((4 * (zext i32 %m to i64) * (sext i32 %stride2 to i64)) + %src) End: ((4 * (zext i32 (1 + %n) to i64))<nuw><nsw> + (4 * (sext i32 %stride2 to i64))<nsw> + %src)
 
 define void @decreasing_outer_iv(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, i32 noundef %stride1, i32 noundef %stride2, i32 noundef %m, i32 noundef %n) {
-; CHECK-LABEL: define void @decreasing_outer_iv
-; CHECK-SAME: (ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[STRIDE1:%.*]], i32 noundef [[STRIDE2:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define void @decreasing_outer_iv(
+; CHECK-SAME: ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[STRIDE1:%.*]], i32 noundef [[STRIDE2:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[CMP21:%.*]] = icmp slt i32 [[M]], 1
 ; CHECK-NEXT:    [[CMP2_NOT18:%.*]] = icmp slt i32 [[N]], 0
 ; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[CMP21]], [[CMP2_NOT18]]
-; CHECK-NEXT:    br i1 [[OR_COND]], label [[EXIT:%.*]], label [[OUTER_LOOP_PRE:%.*]]
-; CHECK:       outer.loop.pre:
+; CHECK-NEXT:    br i1 [[OR_COND]], label %[[EXIT:.*]], label %[[OUTER_LOOP_PRE:.*]]
+; CHECK:       [[OUTER_LOOP_PRE]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add nuw i32 [[N]], 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[M]] to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[STRIDE1]] to i64
@@ -1135,15 +1135,15 @@ define void @decreasing_outer_iv(ptr nocapture noundef %dst, ptr nocapture nound
 ; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP12]], [[TMP7]]
 ; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP13]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = mul nsw i64 [[TMP3]], -4
-; CHECK-NEXT:    br label [[OUTER_LOOP:%.*]]
-; CHECK:       outer.loop:
-; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ [[TMP1]], [[OUTER_LOOP_PRE]] ], [ [[OUTER_IV_NEXT:%.*]], [[INNER_LOOP_EXIT:%.*]] ]
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ [[TMP1]], %[[OUTER_LOOP_PRE]] ], [ [[OUTER_IV_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ]
 ; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nsw i64 [[OUTER_IV]], -1
 ; CHECK-NEXT:    [[TMP15:%.*]] = mul nsw i64 [[OUTER_IV]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul nsw i64 [[OUTER_IV]], [[TMP2]]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
-; CHECK:       vector.memcheck:
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP3]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP1]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
@@ -1151,13 +1151,13 @@ define void @decreasing_outer_iv(ptr nocapture noundef %dst, ptr nocapture nound
 ; CHECK-NEXT:    [[TMP17:%.*]] = or i1 [[FOUND_CONFLICT]], [[STRIDE_CHECK]]
 ; CHECK-NEXT:    [[STRIDE_CHECK4:%.*]] = icmp slt i64 [[TMP14]], 0
 ; CHECK-NEXT:    [[TMP18:%.*]] = or i1 [[TMP17]], [[STRIDE_CHECK4]]
-; CHECK-NEXT:    br i1 [[TMP18]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
+; CHECK-NEXT:    br i1 [[TMP18]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP19:%.*]] = add nsw i64 [[INDEX]], [[TMP15]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP19]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP20]], align 4, !alias.scope [[META53:![0-9]+]]
@@ -1168,15 +1168,15 @@ define void @decreasing_outer_iv(ptr nocapture noundef %dst, ptr nocapture nound
 ; CHECK-NEXT:    store <4 x i32> [[TMP23]], ptr [[TMP22]], align 4, !alias.scope [[META56]], !noalias [[META53]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP58:![0-9]+]]
-; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP58:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_LOOP_EXIT]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_LOOP]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
-; CHECK:       inner.loop:
-; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INNER_IV_NEXT:%.*]], [[INNER_LOOP]] ]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[INNER_LOOP_EXIT]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER_LOOP]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_LOOP]] ]
 ; CHECK-NEXT:    [[TMP25:%.*]] = add nsw i64 [[INNER_IV]], [[TMP15]]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP25]]
 ; CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
@@ -1187,13 +1187,13 @@ define void @decreasing_outer_iv(ptr nocapture noundef %dst, ptr nocapture nound
 ; CHECK-NEXT:    store i32 [[ADD9]], ptr [[ARRAYIDX8]], align 4
 ; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[INNER_LOOP_EXIT]], label [[INNER_LOOP]], !llvm.loop [[LOOP59:![0-9]+]]
-; CHECK:       inner.loop.exit:
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[INNER_LOOP_EXIT]], label %[[INNER_LOOP]], !llvm.loop [[LOOP59:![0-9]+]]
+; CHECK:       [[INNER_LOOP_EXIT]]:
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[OUTER_IV]], 1
-; CHECK-NEXT:    br i1 [[CMP]], label [[OUTER_LOOP]], label [[OUTER_LOOP_EXIT:%.*]]
-; CHECK:       outer.loop.exit:
-; CHECK-NEXT:    br label [[EXIT]]
-; CHECK:       exit:
+; CHECK-NEXT:    br i1 [[CMP]], label %[[OUTER_LOOP]], label %[[OUTER_LOOP_EXIT:.*]]
+; CHECK:       [[OUTER_LOOP_EXIT]]:
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -1263,14 +1263,14 @@ exit:
 ; DEBUG-NEXT: Start: %src End: ((4 * (zext i32 %m to i64) * (zext i32 %n to i64)) + %src)
 
 define void @unknown_inner_stride(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, i32 noundef %stride1, i32 noundef %stride2, i32 noundef %m, i32 noundef %n) {
-; CHECK-LABEL: define void @unknown_inner_stride
-; CHECK-SAME: (ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[STRIDE1:%.*]], i32 noundef [[STRIDE2:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define void @unknown_inner_stride(
+; CHECK-SAME: ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i32 noundef [[STRIDE1:%.*]], i32 noundef [[STRIDE2:%.*]], i32 noundef [[M:%.*]], i32 noundef [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[CMP26:%.*]] = icmp sgt i32 [[M]], 0
 ; CHECK-NEXT:    [[CMP224:%.*]] = icmp sgt i32 [[N]], 0
 ; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[CMP26]], [[CMP224]]
-; CHECK-NEXT:    br i1 [[OR_COND]], label [[OUTER_LOOP_PREHEADER:%.*]], label [[EXIT:%.*]]
-; CHECK:       outer.loop.preheader:
+; CHECK-NEXT:    br i1 [[OR_COND]], label %[[OUTER_LOOP_PREHEADER:.*]], label %[[EXIT:.*]]
+; CHECK:       [[OUTER_LOOP_PREHEADER]]:
 ; CHECK-NEXT:    [[ADD6:%.*]] = add nuw nsw i32 [[N]], 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[STRIDE2]] to i64
 ; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[STRIDE1]] to i64
@@ -1287,29 +1287,29 @@ define void @unknown_inner_stride(ptr nocapture noundef %dst, ptr nocapture noun
 ; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[WIDE_TRIP_COUNT]], [[WIDE_TRIP_COUNT39]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = shl i64 [[TMP9]], 2
 ; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP10]]
-; CHECK-NEXT:    br label [[OUTER_LOOP:%.*]]
-; CHECK:       outer.loop:
-; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, [[OUTER_LOOP_PREHEADER]] ], [ [[OUTER_IV_NEXT:%.*]], [[INNER_LOOP_EXIT:%.*]] ]
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[OUTER_LOOP_PREHEADER]] ], [ [[OUTER_IV_NEXT:%.*]], %[[INNER_LOOP_EXIT:.*]] ]
 ; CHECK-NEXT:    [[TMP11:%.*]] = mul nsw i64 [[OUTER_IV]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = mul nsw i64 [[OUTER_IV]], [[TMP3]]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
-; CHECK:       vector.scevcheck:
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK:       [[VECTOR_SCEVCHECK]]:
 ; CHECK-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i32 [[STRIDE1]], 1
 ; CHECK-NEXT:    [[IDENT_CHECK1:%.*]] = icmp ne i32 [[STRIDE2]], 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = or i1 [[IDENT_CHECK]], [[IDENT_CHECK1]]
-; CHECK-NEXT:    br i1 [[TMP13]], label [[SCALAR_PH]], label [[VECTOR_MEMCHECK:%.*]]
-; CHECK:       vector.memcheck:
+; CHECK-NEXT:    br i1 [[TMP13]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP2]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP14:%.*]] = add nsw i64 [[INDEX]], [[TMP11]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP14]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP15]], align 4, !alias.scope [[META60:![0-9]+]]
@@ -1320,15 +1320,15 @@ define void @unknown_inner_stride(ptr nocapture noundef %dst, ptr nocapture noun
 ; CHECK-NEXT:    store <4 x i32> [[TMP18]], ptr [[TMP17]], align 4, !alias.scope [[META63]], !noalias [[META60]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP65:![0-9]+]]
-; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP65:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_LOOP_EXIT]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_LOOP]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
-; CHECK:       inner.loop:
-; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INNER_IV_NEXT:%.*]], [[INNER_LOOP]] ]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[INNER_LOOP_EXIT]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER_LOOP]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_LOOP]] ]
 ; CHECK-NEXT:    [[TMP20:%.*]] = mul nsw i64 [[INNER_IV]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = add nsw i64 [[TMP20]], [[TMP11]]
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP21]]
@@ -1341,14 +1341,14 @@ define void @unknown_inner_stride(ptr nocapture noundef %dst, ptr nocapture noun
 ; CHECK-NEXT:    store i32 [[ADD12_US]], ptr [[ARRAYIDX11_US]], align 4
 ; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[INNER_LOOP_EXIT]], label [[INNER_LOOP]], !llvm.loop [[LOOP66:![0-9]+]]
-; CHECK:       inner.loop.exit:
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[INNER_LOOP_EXIT]], label %[[INNER_LOOP]], !llvm.loop [[LOOP66:![0-9]+]]
+; CHECK:       [[INNER_LOOP_EXIT]]:
 ; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND40_NOT:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[WIDE_TRIP_COUNT39]]
-; CHECK-NEXT:    br i1 [[EXITCOND40_NOT]], label [[EXIT_LOOPEXIT:%.*]], label [[OUTER_LOOP]]
-; CHECK:       exit.loopexit:
-; CHECK-NEXT:    br label [[EXIT]]
-; CHECK:       exit:
+; CHECK-NEXT:    br i1 [[EXITCOND40_NOT]], label %[[EXIT_LOOPEXIT:.*]], label %[[OUTER_LOOP]]
+; CHECK:       [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -1403,27 +1403,27 @@ exit:
 ; of the outer loop as start value. It is sufficient to subtract the start
 ; values (%dst, %src) of the outer AddRecs.
 define void @nested_loop_start_of_inner_ptr_addrec_is_same_outer_addrec(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, i64 noundef %m, i64 noundef %n) {
-; CHECK-LABEL: define void @nested_loop_start_of_inner_ptr_addrec_is_same_outer_addrec
-; CHECK-SAME: (ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i64 noundef [[M:%.*]], i64 noundef [[N:%.*]]) {
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define void @nested_loop_start_of_inner_ptr_addrec_is_same_outer_addrec(
+; CHECK-SAME: ptr noundef captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC:%.*]], i64 noundef [[M:%.*]], i64 noundef [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[SRC2:%.*]] = ptrtoaddr ptr [[SRC]] to i64
 ; CHECK-NEXT:    [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[DST1]], [[SRC2]]
-; CHECK-NEXT:    br label [[OUTER_LOOP:%.*]]
-; CHECK:       outer.loop:
-; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[OUTER_IV_NEXT:%.*]], [[INNER_EXIT:%.*]] ]
+; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
+; CHECK:       [[OUTER_LOOP]]:
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[INNER_EXIT:.*]] ]
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i64 [[OUTER_IV]], [[N]]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
-; CHECK:       vector.memcheck:
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 16
-; CHECK-NEXT:    br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
+; CHECK-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[INDEX]], [[MUL]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP1]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
@@ -1432,15 +1432,15 @@ define void @nested_loop_start_of_inner_ptr_addrec_is_same_outer_addrec(ptr noca
 ; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr [[TMP3]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP67:![0-9]+]]
-; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP67:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[INNER_EXIT]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTER_LOOP]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
-; CHECK:       inner.loop:
-; CHECK-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_INNER_NEXT:%.*]], [[INNER_LOOP]] ]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[INNER_EXIT]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER_LOOP]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[INNER_LOOP:.*]]
+; CHECK:       [[INNER_LOOP]]:
+; CHECK-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_INNER_NEXT:%.*]], %[[INNER_LOOP]] ]
 ; CHECK-NEXT:    [[IDX:%.*]] = add nuw nsw i64 [[IV_INNER]], [[MUL]]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IDX]]
 ; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[GEP_SRC]], align 4
@@ -1449,12 +1449,12 @@ define void @nested_loop_start_of_inner_ptr_addrec_is_same_outer_addrec(ptr noca
 ; CHECK-NEXT:    store i32 [[ADD]], ptr [[GEP_DST]], align 4
 ; CHECK-NEXT:    [[IV_INNER_NEXT]] = add nuw nsw i64 [[IV_INNER]], 1
 ; CHECK-NEXT:    [[INNER_EXIT_COND:%.*]] = icmp eq i64 [[IV_INNER_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[INNER_EXIT_COND]], label [[INNER_EXIT]], label [[INNER_LOOP]], !llvm.loop [[LOOP68:![0-9]+]]
-; CHECK:       inner.exit:
+; CHECK-NEXT:    br i1 [[INNER_EXIT_COND]], label %[[INNER_EXIT]], label %[[INNER_LOOP]], !llvm.loop [[LOOP68:![0-9]+]]
+; CHECK:       [[INNER_EXIT]]:
 ; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
 ; CHECK-NEXT:    [[OUTER_EXIT_COND:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[M]]
-; CHECK-NEXT:    br i1 [[OUTER_EXIT_COND]], label [[OUTER_EXIT:%.*]], label [[OUTER_LOOP]]
-; CHECK:       outer.exit:
+; CHECK-NEXT:    br i1 [[OUTER_EXIT_COND]], label %[[OUTER_EXIT:.*]], label %[[OUTER_LOOP]]
+; CHECK:       [[OUTER_EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -1489,56 +1489,56 @@ outer.exit:
 ; The stride for the access in the inner loop is known to be non-negative via
 ; loop guards.
 define void @stride_check_known_via_loop_guard(ptr %C, ptr %A, i32 %Acols) {
-; CHECK-LABEL: define void @stride_check_known_via_loop_guard
-; CHECK-SAME: (ptr [[C:%.*]], ptr [[A:%.*]], i32 [[ACOLS:%.*]]) {
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define void @stride_check_known_via_loop_guard(
+; CHECK-SAME: ptr [[C:%.*]], ptr [[A:%.*]], i32 [[ACOLS:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[PRE_C:%.*]] = icmp ugt i32 [[ACOLS]], 0
-; CHECK-NEXT:    br i1 [[PRE_C]], label [[EXIT:%.*]], label [[OUTER_HEADER_PREHEADER:%.*]]
-; CHECK:       outer.header.preheader:
+; CHECK-NEXT:    br i1 [[PRE_C]], label %[[EXIT:.*]], label %[[OUTER_HEADER_PREHEADER:.*]]
+; CHECK:       [[OUTER_HEADER_PREHEADER]]:
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[C]], i64 8000
 ; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 8
-; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
-; CHECK:       outer.header:
-; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i32 [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ], [ 0, [[OUTER_HEADER_PREHEADER]] ]
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i32 [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ], [ 0, %[[OUTER_HEADER_PREHEADER]] ]
 ; CHECK-NEXT:    [[MUL_US:%.*]] = mul i32 [[OUTER_IV]], [[ACOLS]]
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr double, ptr [[A]], i32 [[MUL_US]]
-; CHECK-NEXT:    br label [[VECTOR_MEMCHECK:%.*]]
-; CHECK:       vector.memcheck:
+; CHECK-NEXT:    br label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[C]], [[SCEVGEP1]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[ARRAYIDX_US]], align 8, !alias.scope [[META69:![0-9]+]]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[TMP0]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds double, ptr [[C]], i32 [[INDEX]]
 ; CHECK-NEXT:    store <4 x double> [[BROADCAST_SPLAT]], ptr [[TMP1]], align 8, !alias.scope [[META72:![0-9]+]], !noalias [[META69]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP74:![0-9]+]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    br label [[OUTER_LATCH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    br label [[INNER:%.*]]
-; CHECK:       inner:
-; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[INNER_IV_NEXT:%.*]], [[INNER]] ]
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP74:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[OUTER_LATCH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    br label %[[INNER:.*]]
+; CHECK:       [[INNER]]:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER]] ]
 ; CHECK-NEXT:    [[GEP_C:%.*]] = getelementptr inbounds double, ptr [[C]], i32 [[INNER_IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load double, ptr [[ARRAYIDX_US]], align 8
 ; CHECK-NEXT:    store double [[L]], ptr [[GEP_C]], align 8
 ; CHECK-NEXT:    [[INNER_IV_NEXT]] = add i32 [[INNER_IV]], 1
 ; CHECK-NEXT:    [[INNER_C:%.*]] = icmp eq i32 [[INNER_IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[INNER_C]], label [[OUTER_LATCH]], label [[INNER]], !llvm.loop [[LOOP75:![0-9]+]]
-; CHECK:       outer.latch:
+; CHECK-NEXT:    br i1 [[INNER_C]], label %[[OUTER_LATCH]], label %[[INNER]], !llvm.loop [[LOOP75:![0-9]+]]
+; CHECK:       [[OUTER_LATCH]]:
 ; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add i32 [[OUTER_IV]], 1
 ; CHECK-NEXT:    [[OUTER_C:%.*]] = icmp ult i32 [[OUTER_IV]], 128
-; CHECK-NEXT:    br i1 [[OUTER_C]], label [[EXIT_LOOPEXIT:%.*]], label [[OUTER_HEADER]]
-; CHECK:       exit.loopexit:
-; CHECK-NEXT:    br label [[EXIT]]
-; CHECK:       exit:
+; CHECK-NEXT:    br i1 [[OUTER_C]], label %[[EXIT_LOOPEXIT:.*]], label %[[OUTER_HEADER]]
+; CHECK:       [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
index 2275aef2b4f97..ad991e5170473 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
@@ -202,10 +202,8 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) {
 ; CHECK-VF4UF1-NEXT:    [[WIDE_LOAD]] = load <vscale x 4 x i32>, ptr [[TMP10]], align 4
 ; CHECK-VF4UF1-NEXT:    [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_LOAD]], i32 1)
 ; CHECK-VF4UF1-NEXT:    [[TMP13:%.*]] = sub nsw <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP12]]
-; CHECK-VF4UF1-NEXT:    [[TMP14:%.*]] = icmp sgt <vscale x 4 x i32> [[TMP13]], zeroinitializer
-; CHECK-VF4UF1-NEXT:    [[TMP15:%.*]] = select <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i32> [[TMP13]], <vscale x 4 x i32> zeroinitializer
-; CHECK-VF4UF1-NEXT:    [[TMP16:%.*]] = icmp slt <vscale x 4 x i32> [[VEC_PHI]], [[TMP15]]
-; CHECK-VF4UF1-NEXT:    [[TMP17]] = select <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP15]]
+; CHECK-VF4UF1-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[TMP13]], <vscale x 4 x i32> zeroinitializer)
+; CHECK-VF4UF1-NEXT:    [[TMP17]] = call <vscale x 4 x i32> @llvm.smin.nxv4i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP14]])
 ; CHECK-VF4UF1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; CHECK-VF4UF1-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF4UF1-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -266,14 +264,10 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) {
 ; CHECK-VF4UF2-NEXT:    [[TMP16:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[WIDE_LOAD2]], i32 1)
 ; CHECK-VF4UF2-NEXT:    [[TMP17:%.*]] = sub nsw <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP15]]
 ; CHECK-VF4UF2-NEXT:    [[TMP18:%.*]] = sub nsw <vscale x 4 x i32> [[WIDE_LOAD2]], [[TMP16]]
-; CHECK-VF4UF2-NEXT:    [[TMP19:%.*]] = icmp sgt <vscale x 4 x i32> [[TMP17]], zeroinitializer
-; CHECK-VF4UF2-NEXT:    [[TMP20:%.*]] = icmp sgt <vscale x 4 x i32> [[TMP18]], zeroinitializer
-; CHECK-VF4UF2-NEXT:    [[TMP21:%.*]] = select <vscale x 4 x i1> [[TMP19]], <vscale x 4 x i32> [[TMP17]], <vscale x 4 x i32> zeroinitializer
-; CHECK-VF4UF2-NEXT:    [[TMP22:%.*]] = select <vscale x 4 x i1> [[TMP20]], <vscale x 4 x i32> [[TMP18]], <vscale x 4 x i32> zeroinitializer
-; CHECK-VF4UF2-NEXT:    [[TMP23:%.*]] = icmp slt <vscale x 4 x i32> [[VEC_PHI]], [[TMP21]]
-; CHECK-VF4UF2-NEXT:    [[TMP24:%.*]] = icmp slt <vscale x 4 x i32> [[VEC_PHI1]], [[TMP22]]
-; CHECK-VF4UF2-NEXT:    [[TMP25]] = select <vscale x 4 x i1> [[TMP23]], <vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP21]]
-; CHECK-VF4UF2-NEXT:    [[TMP26]] = select <vscale x 4 x i1> [[TMP24]], <vscale x 4 x i32> [[VEC_PHI1]], <vscale x 4 x i32> [[TMP22]]
+; CHECK-VF4UF2-NEXT:    [[TMP19:%.*]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[TMP17]], <vscale x 4 x i32> zeroinitializer)
+; CHECK-VF4UF2-NEXT:    [[TMP20:%.*]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[TMP18]], <vscale x 4 x i32> zeroinitializer)
+; CHECK-VF4UF2-NEXT:    [[TMP25]] = call <vscale x 4 x i32> @llvm.smin.nxv4i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP19]])
+; CHECK-VF4UF2-NEXT:    [[TMP26]] = call <vscale x 4 x i32> @llvm.smin.nxv4i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 4 x i32> [[TMP20]])
 ; CHECK-VF4UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; CHECK-VF4UF2-NEXT:    [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF4UF2-NEXT:    br i1 [[TMP27]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -320,10 +314,8 @@ scalar.body:
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
   %1 = load i32, ptr %arrayidx, align 4
   %sub3 = sub nsw i32 %1, %0
-  %cmp4 = icmp sgt i32 %sub3, 0
-  %cond = select i1 %cmp4, i32 %sub3, i32 0
-  %cmp5 = icmp slt i32 %minmax.028, %cond
-  %minmax.0.cond = select i1 %cmp5, i32 %minmax.028, i32 %cond
+  %cond = call i32 @llvm.smax(i32 %sub3, i32 0)
+  %minmax.0.cond = call i32 @llvm.smin(i32 %minmax.028, i32 %cond)
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp eq i32 %lftr.wideiv, %n
diff --git a/llvm/test/Transforms/LoopVectorize/select-reduction.ll b/llvm/test/Transforms/LoopVectorize/select-reduction.ll
index 28bb8a2b13114..c69ef5299fe6c 100644
--- a/llvm/test/Transforms/LoopVectorize/select-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-reduction.ll
@@ -23,18 +23,18 @@ define i32 @test(i64 %N, i32 %x) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[VEC_PHI]], splat (i32 10)
-; CHECK-NEXT:    [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i32> [[VEC_PHI]], <4 x i32> splat (i32 10)
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP1]] = select <4 x i1> [[TMP0]], <4 x i1> splat (i1 true), <4 x i1> [[VEC_PHI]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IV]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[TMP6]], <4 x i32> [[TMP2]], <4 x i32> [[VEC_PHI]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP3]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = freeze i1 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP6]], i32 10, i32 0
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       exit.loopexit:
 ; CHECK-NEXT:    br label [[EXIT]]
diff --git a/llvm/test/Transforms/LowerSwitch/do-not-handle-impossible-values.ll b/llvm/test/Transforms/LowerSwitch/do-not-handle-impossible-values.ll
index ed2204fa807b3..4c77b61607607 100644
--- a/llvm/test/Transforms/LowerSwitch/do-not-handle-impossible-values.ll
+++ b/llvm/test/Transforms/LowerSwitch/do-not-handle-impossible-values.ll
@@ -603,7 +603,7 @@ define void @test12(i1 %arg) {
 ; CHECK-NEXT:    br label [[LATCH]]
 ; CHECK:       latch:
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[INDVAR]], 1
-; CHECK-NEXT:    br i1 %arg, label [[EXIT:%.*]], label [[FOR_BODY]]
+; CHECK-NEXT:    br i1 [[ARG:%.*]], label [[EXIT:%.*]], label [[FOR_BODY]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -801,10 +801,8 @@ define i32 @test16(float %f) {
 ; CHECK-LABEL: @test16(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[I:%.*]] = fptosi float [[F:%.*]] to i64
-; CHECK-NEXT:    [[COND_LEFT:%.*]] = icmp slt i64 [[I]], 0
-; CHECK-NEXT:    [[CLAMP_LEFT:%.*]] = select i1 [[COND_LEFT]], i64 0, i64 [[I]]
-; CHECK-NEXT:    [[COND_RIGHT:%.*]] = icmp sgt i64 [[I]], 3
-; CHECK-NEXT:    [[CLAMP:%.*]] = select i1 [[COND_RIGHT]], i64 3, i64 [[CLAMP_LEFT]]
+; CHECK-NEXT:    [[CLAMP_LEFT:%.*]] = call i64 @llvm.smax.i64(i64 [[I]], i64 0)
+; CHECK-NEXT:    [[CLAMP:%.*]] = call i64 @llvm.smin.i64(i64 [[CLAMP_LEFT]], i64 3)
 ; CHECK-NEXT:    br label [[LEAFBLOCK:%.*]]
 ; CHECK:       LeafBlock:
 ; CHECK-NEXT:    [[SWITCHLEAF:%.*]] = icmp sge i64 [[CLAMP]], 2
@@ -821,10 +819,8 @@ define i32 @test16(float %f) {
 ;
 entry:
   %i = fptosi float %f to i64
-  %cond.left = icmp slt i64 %i, 0
-  %clamp.left = select i1 %cond.left, i64 0, i64 %i
-  %cond.right = icmp sgt i64 %i, 3
-  %clamp = select i1 %cond.right, i64 3, i64 %clamp.left
+  %clamp.left = call i64 @llvm.smax.i64(i64 %i, i64 0)
+  %clamp = call i64 @llvm.smin.i64(i64 %clamp.left, i64 3)
   switch i64 %clamp, label %case.D [
   i64 0, label %case.1
   i64 1, label %case.1
diff --git a/llvm/test/Transforms/MergeFunc/noipa.ll b/llvm/test/Transforms/MergeFunc/noipa.ll
new file mode 100644
index 0000000000000..d1fa274f7808f
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/noipa.ll
@@ -0,0 +1,53 @@
+; RUN: opt -passes=mergefunc -S < %s | FileCheck %s
+
+; MergeFunctions merging a `noipa` function with an identical one.
+; If `noipa` is working, MergeFunctions should NOT merge
+; @foo_noipa into @baz_noipa (or vice versa), because it inspects
+; the function definitions.
+
+; CHECK-NOT: define internal i32 @foo(
+define internal i32 @foo(i32 %x, i32 %y) noinline {
+  %sum = add i32 %x, %y
+  %mul = mul i32 %sum, %x
+  %sub = sub i32 %mul, %y
+  ret i32 %sub
+}
+
+; CHECK: define internal i32 @baz(
+define internal i32 @baz(i32 %x, i32 %y) noinline {
+  %sum = add i32 %x, %y
+  %mul = mul i32 %sum, %x
+  %sub = sub i32 %mul, %y
+  ret i32 %sub
+}
+
+; CHECK: define internal i32 @foo_noipa(
+define internal i32 @foo_noipa(i32 %x, i32 %y) noipa noinline {
+  %sum = add i32 %x, %y
+  %mul = mul i32 %sum, %x
+  %sub = sub i32 %mul, %y
+  ret i32 %sub
+}
+
+; CHECK: define internal i32 @baz_noipa(
+define internal i32 @baz_noipa(i32 %x, i32 %y) noipa noinline {
+  %sum = add i32 %x, %y
+  %mul = mul i32 %sum, %x
+  %sub = sub i32 %mul, %y
+  ret i32 %sub
+}
+
+define i32 @bar() {
+  ; CHECK: call i32 @baz(
+  %r1 = call i32 @foo(i32 1, i32 2)
+  ; CHECK: call i32 @baz(
+  %r2 = call i32 @baz(i32 3, i32 4)
+  ; CHECK: call i32 @foo_noipa(
+  %r3 = call i32 @foo_noipa(i32 1, i32 2)
+  ; CHECK: call i32 @baz_noipa(
+  %r4 = call i32 @baz_noipa(i32 3, i32 4)
+  %res1 = add i32 %r1, %r2
+  %res2 = add i32 %r3, %r4
+  %res3 = add i32 %res1, %res2
+  ret i32 %res3
+}
diff --git a/llvm/test/Transforms/NaryReassociate/nary-gep-zero-sized-element.ll b/llvm/test/Transforms/NaryReassociate/nary-gep-zero-sized-element.ll
new file mode 100644
index 0000000000000..f78023684d11f
--- /dev/null
+++ b/llvm/test/Transforms/NaryReassociate/nary-gep-zero-sized-element.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=nary-reassociate -S | FileCheck %s
+
+; This test checks that NaryReassociate does not crash when a GEP's result
+; element type has a size of zero (eg. [0 x ptr] such as `%T` below).
+
+; Struct with a zero-sized field: sizeof(field 1) == 0.
+%T = type { [1 x ptr], [0 x ptr] }
+
+define ptr @foo(ptr %base, i64 %x, i64 %y) {
+; CHECK-LABEL: define ptr @foo(
+; CHECK-SAME: ptr [[BASE:%.*]], i64 [[X:%.*]], i64 [[Y:%.*]]) {
+; CHECK-NEXT:    [[Z:%.*]] = add i64 [[X]], [[Y]]
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr [[T:%.*]], ptr [[BASE]], i64 [[X]], i32 1
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds [[T]], ptr [[BASE]], i64 [[Z]], i32 1
+; CHECK-NEXT:    ret ptr [[GEP2]]
+;
+  %z = add i64 %x, %y
+  %gep1 = getelementptr inbounds %T, ptr %base, i64 %x, i32 1
+  %gep2 = getelementptr inbounds %T, ptr %base, i64 %z, i32 1
+  ret ptr %gep2
+}
diff --git a/llvm/test/Transforms/NaryReassociate/nary-req.ll b/llvm/test/Transforms/NaryReassociate/nary-req.ll
index 054afd7d44e00..8f339769831fd 100644
--- a/llvm/test/Transforms/NaryReassociate/nary-req.ll
+++ b/llvm/test/Transforms/NaryReassociate/nary-req.ll
@@ -9,22 +9,16 @@ declare i64 @llvm.umin.i64(i64, i64)
 ; use from %smax3 and side use from %res2.
 define i32 @smax_test1(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @smax_test1(
-; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SMAX1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[C2:%.*]] = icmp sgt i32 [[B]], [[C:%.*]]
-; CHECK-NEXT:    [[SMAX2:%.*]] = select i1 [[C2]], i32 [[B]], i32 [[C]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp sgt i32 [[SMAX2]], [[A]]
-; CHECK-NEXT:    [[SMAX3:%.*]] = select i1 [[C3]], i32 [[SMAX2]], i32 [[A]]
+; CHECK-NEXT:    [[SMAX1:%.*]] = call i32 @llvm.smax.i32(i32 [[A:%.*]], i32 [[B:%.*]])
+; CHECK-NEXT:    [[SMAX2:%.*]] = call i32 @llvm.smax.i32(i32 [[B]], i32 [[C:%.*]])
+; CHECK-NEXT:    [[SMAX3:%.*]] = call i32 @llvm.smax.i32(i32 [[SMAX2]], i32 [[A]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMAX1]], [[SMAX3]]
 ; CHECK-NEXT:    [[RES2:%.*]] = add i32 [[RES]], [[SMAX2]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
-  %c1 = icmp sgt i32 %a, %b
-  %smax1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp sgt i32 %b, %c
-  %smax2 = select i1 %c2, i32 %b, i32 %c
-  %c3 = icmp sgt i32 %smax2, %a
-  %smax3 = select i1 %c3, i32 %smax2, i32 %a
+  %smax1 = call i32 @llvm.smax.i32(i32 %a, i32 %b)
+  %smax2 = call i32 @llvm.smax.i32(i32 %b, i32 %c)
+  %smax3 = call i32 @llvm.smax.i32(i32 %smax2, i32 %a)
   %res = add i32 %smax1, %smax3
   %res2 = add i32 %res, %smax2
   ret i32 %res
@@ -71,33 +65,18 @@ bb:
 ; which is matched on the next iteration.
 define i32 @nary_infinite_loop_minmax(i32 %d0, i32 %d1, i32 %d2, i32 %d3) {
 ; CHECK-LABEL: @nary_infinite_loop_minmax(
-; CHECK-NEXT:    [[CMP0:%.*]] = icmp slt i32 [[D2:%.*]], [[D1:%.*]]
-; CHECK-NEXT:    [[SEL0:%.*]] = select i1 [[CMP0]], i32 [[D1]], i32 [[D2]]
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[D3:%.*]], [[D0:%.*]]
-; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[CMP1]], i32 [[D0]], i32 [[D3]]
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[SEL1]], [[SEL0]]
-; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[CMP2]], i32 [[SEL1]], i32 [[SEL0]]
-; CHECK-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[D3]], [[D0]]
-; CHECK-NEXT:    [[SEL3:%.*]] = select i1 [[CMP3]], i32 [[D0]], i32 [[D3]]
-; CHECK-NEXT:    [[SEL5_NARY:%.*]] = call i32 @llvm.smax.i32(i32 [[SEL0]], i32 [[SEL3]])
+; CHECK-NEXT:    [[SEL0:%.*]] = call i32 @llvm.smax.i32(i32 [[D2:%.*]], i32 [[D1:%.*]])
+; CHECK-NEXT:    [[SEL1:%.*]] = call i32 @llvm.smax.i32(i32 [[D3:%.*]], i32 [[D0:%.*]])
+; CHECK-NEXT:    [[SEL2:%.*]] = call i32 @llvm.smax.i32(i32 [[SEL1]], i32 [[SEL0]])
+; CHECK-NEXT:    [[SEL3:%.*]] = call i32 @llvm.smax.i32(i32 [[D3]], i32 [[D0]])
+; CHECK-NEXT:    [[SEL5_NARY:%.*]] = call i32 @llvm.smax.i32(i32 [[SEL3]], i32 [[SEL0]])
 ; CHECK-NEXT:    ret i32 [[SEL5_NARY]]
 ;
-  %cmp0 = icmp slt i32 %d2, %d1
-  %sel0 = select i1 %cmp0, i32 %d1, i32 %d2
-
-  %cmp1 = icmp slt i32 %d3, %d0
-  %sel1 = select i1 %cmp1, i32 %d0, i32 %d3
-
-  %cmp2 = icmp slt i32 %sel1, %sel0
-  %sel2 = select i1 %cmp2, i32 %sel1, i32 %sel0
-
-  %cmp3 = icmp slt i32 %d3, %d0
-  %sel3 = select i1 %cmp3, i32 %d0, i32 %d3
-
-  %cmp4 = icmp slt i32 %sel3, %d2
-  %sel4 = select i1 %cmp4, i32 %d2, i32 %sel3
-
-  %cmp5 = icmp slt i32 %sel4, %d1
-  %sel5 = select i1 %cmp5, i32 %d1, i32 %sel4
+  %sel0 = call i32 @llvm.smax.i32(i32 %d2, i32 %d1)
+  %sel1 = call i32 @llvm.smax.i32(i32 %d3, i32 %d0)
+  %sel2 = call i32 @llvm.smax.i32(i32 %sel1, i32 %sel0)
+  %sel3 = call i32 @llvm.smax.i32(i32 %d3, i32 %d0)
+  %sel4 = call i32 @llvm.smax.i32(i32 %sel3, i32 %d2)
+  %sel5 = call i32 @llvm.smax.i32(i32 %sel4, i32 %d1)
   ret i32 %sel5
 }
diff --git a/llvm/test/Transforms/NaryReassociate/nary-smax.ll b/llvm/test/Transforms/NaryReassociate/nary-smax.ll
index 43e93c7d2a08f..4889039a4e271 100644
--- a/llvm/test/Transforms/NaryReassociate/nary-smax.ll
+++ b/llvm/test/Transforms/NaryReassociate/nary-smax.ll
@@ -7,18 +7,14 @@ declare i32 @llvm.smax.i32(i32 %a, i32 %b)
 ; m2 = smax(smax((b,c), a) -> m2 = smax(m1, c)
 define i32 @smax_test1(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @smax_test1(
-; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SMAX1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    [[SMAX1:%.*]] = call i32 @llvm.smax.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    [[SMAX3_NARY:%.*]] = call i32 @llvm.smax.i32(i32 [[SMAX1]], i32 [[C:%.*]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMAX1]], [[SMAX3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
-  %c1 = icmp sgt i32 %a, %b
-  %smax1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp sgt i32 %b, %c
-  %smax2 = select i1 %c2, i32 %b, i32 %c
-  %c3 = icmp sgt i32 %smax2, %a
-  %smax3 = select i1 %c3, i32 %smax2, i32 %a
+  %smax1 = call i32 @llvm.smax.i32(i32 %a, i32 %b)
+  %smax2 = call i32 @llvm.smax.i32(i32 %b, i32 %c)
+  %smax3 = call i32 @llvm.smax.i32(i32 %smax2, i32 %a)
   %res = add i32 %smax1, %smax3
   ret i32 %res
 }
@@ -27,33 +23,14 @@ define i32 @smax_test1(i32 %a, i32 %b, i32 %c) {
 ; m2 = smax(b, (smax(a, c))) -> m2 = smax(m1, c)
 define i32 @smax_test2(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @smax_test2(
-; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SMAX1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[SMAX3_NARY:%.*]] = call i32 @llvm.smax.i32(i32 [[SMAX1]], i32 [[C:%.*]])
-; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMAX1]], [[SMAX3_NARY]]
-; CHECK-NEXT:    ret i32 [[RES]]
-;
-  %c1 = icmp sgt i32 %a, %b
-  %smax1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp sgt i32 %a, %c
-  %smax2 = select i1 %c2, i32 %a, i32 %c
-  %c3 = icmp sgt i32 %b, %smax2
-  %smax3 = select i1 %c3, i32 %b, i32 %smax2
-  %res = add i32 %smax1, %smax3
-  ret i32 %res
-}
-
-; Same test as smax_test1 but uses @llvm.smax intrinsic
-define i32 @smax_test3(i32 %a, i32 %b, i32 %c) {
-; CHECK-LABEL: @smax_test3(
 ; CHECK-NEXT:    [[SMAX1:%.*]] = call i32 @llvm.smax.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    [[SMAX3_NARY:%.*]] = call i32 @llvm.smax.i32(i32 [[SMAX1]], i32 [[C:%.*]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMAX1]], [[SMAX3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %smax1 = call i32 @llvm.smax.i32(i32 %a, i32 %b)
-  %smax2 = call i32 @llvm.smax.i32(i32 %b, i32 %c)
-  %smax3 = call i32 @llvm.smax.i32(i32 %smax2, i32 %a)
+  %smax2 = call i32 @llvm.smax.i32(i32 %a, i32 %c)
+  %smax3 = call i32 @llvm.smax.i32(i32 %b, i32 %smax2)
   %res = add i32 %smax1, %smax3
   ret i32 %res
 }
@@ -62,18 +39,14 @@ define i32 @smax_test3(i32 %a, i32 %b, i32 %c) {
 ; m2 = smax(smax_or_eq((b,c), a) -> m2 = smax(m1, c)
 define i32 @umax_test4(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @umax_test4(
-; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SMAX1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    [[SMAX1:%.*]] = call i32 @llvm.smax.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    [[SMAX3_NARY:%.*]] = call i32 @llvm.smax.i32(i32 [[SMAX1]], i32 [[C:%.*]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMAX1]], [[SMAX3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
-  %c1 = icmp sgt i32 %a, %b
-  %smax1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp sge i32 %b, %c
-  %smax_or_eq2 = select i1 %c2, i32 %b, i32 %c
-  %c3 = icmp sgt i32 %smax_or_eq2, %a
-  %smax3 = select i1 %c3, i32 %smax_or_eq2, i32 %a
+  %smax1 = call i32 @llvm.smax.i32(i32 %a, i32 %b)
+  %smax_or_eq2 = call i32 @llvm.smax.i32(i32 %b, i32 %c)
+  %smax3 = call i32 @llvm.smax.i32(i32 %smax_or_eq2, i32 %a)
   %res = add i32 %smax1, %smax3
   ret i32 %res
 }
@@ -82,18 +55,14 @@ define i32 @umax_test4(i32 %a, i32 %b, i32 %c) {
 ; m2 = smax_or_eq(smax((b,c), a) -> m2 = smax(m1, c)
 define i32 @smax_test5(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @smax_test5(
-; CHECK-NEXT:    [[C1:%.*]] = icmp sge i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SMAX_OR_EQ1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    [[SMAX_OR_EQ1:%.*]] = call i32 @llvm.smax.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    [[SMAX_OR_EQ3_NARY:%.*]] = call i32 @llvm.smax.i32(i32 [[SMAX_OR_EQ1]], i32 [[C:%.*]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMAX_OR_EQ1]], [[SMAX_OR_EQ3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
-  %c1 = icmp sge i32 %a, %b
-  %smax_or_eq1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp sgt i32 %b, %c
-  %smax2 = select i1 %c2, i32 %b, i32 %c
-  %c3 = icmp sge i32 %smax2, %a
-  %smax_or_eq3 = select i1 %c3, i32 %smax2, i32 %a
+  %smax_or_eq1 = call i32 @llvm.smax.i32(i32 %a, i32 %b)
+  %smax2 = call i32 @llvm.smax.i32(i32 %b, i32 %c)
+  %smax_or_eq3 = call i32 @llvm.smax.i32(i32 %smax2, i32 %a)
   %res = add i32 %smax_or_eq1, %smax_or_eq3
   ret i32 %res
 }
@@ -102,21 +71,15 @@ define i32 @smax_test5(i32 %a, i32 %b, i32 %c) {
 ; m2 = smax(umax((b,c), a) ; check that signed and unsigned maxs are not mixed
 define i32 @smax_test6(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @smax_test6(
-; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SMAX1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[C2:%.*]] = icmp ugt i32 [[B]], [[C:%.*]]
-; CHECK-NEXT:    [[UMAX2:%.*]] = select i1 [[C2]], i32 [[B]], i32 [[C]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp sgt i32 [[UMAX2]], [[A]]
-; CHECK-NEXT:    [[SMAX3:%.*]] = select i1 [[C3]], i32 [[UMAX2]], i32 [[A]]
+; CHECK-NEXT:    [[SMAX1:%.*]] = call i32 @llvm.smax.i32(i32 [[A:%.*]], i32 [[B:%.*]])
+; CHECK-NEXT:    [[UMAX2:%.*]] = call i32 @llvm.umax.i32(i32 [[B]], i32 [[C:%.*]])
+; CHECK-NEXT:    [[SMAX3:%.*]] = call i32 @llvm.smax.i32(i32 [[UMAX2]], i32 [[A]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMAX1]], [[SMAX3]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
-  %c1 = icmp sgt i32 %a, %b
-  %smax1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp ugt i32 %b, %c
-  %umax2 = select i1 %c2, i32 %b, i32 %c
-  %c3 = icmp sgt i32 %umax2, %a
-  %smax3 = select i1 %c3, i32 %umax2, i32 %a
+  %smax1 = call i32 @llvm.smax.i32(i32 %a, i32 %b)
+  %umax2 = call i32 @llvm.umax.i32(i32 %b, i32 %c)
+  %smax3 = call i32 @llvm.smax.i32(i32 %umax2, i32 %a)
   %res = add i32 %smax1, %smax3
   ret i32 %res
 }
@@ -125,21 +88,15 @@ define i32 @smax_test6(i32 %a, i32 %b, i32 %c) {
 ; m2 = smax(smin((b,c), a) ; check that max and min are not mixed
 define i32 @smax_test7(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @smax_test7(
-; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SMAX1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[C2:%.*]] = icmp slt i32 [[B]], [[C:%.*]]
-; CHECK-NEXT:    [[SMIN2:%.*]] = select i1 [[C2]], i32 [[B]], i32 [[C]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp slt i32 [[SMIN2]], [[A]]
-; CHECK-NEXT:    [[SMAX3:%.*]] = select i1 [[C3]], i32 [[SMIN2]], i32 [[A]]
+; CHECK-NEXT:    [[SMAX1:%.*]] = call i32 @llvm.smax.i32(i32 [[A:%.*]], i32 [[B:%.*]])
+; CHECK-NEXT:    [[SMIN2:%.*]] = call i32 @llvm.smin.i32(i32 [[B]], i32 [[C:%.*]])
+; CHECK-NEXT:    [[SMAX3:%.*]] = call i32 @llvm.smin.i32(i32 [[SMIN2]], i32 [[A]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMAX1]], [[SMAX3]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
-  %c1 = icmp sgt i32 %a, %b
-  %smax1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp slt i32 %b, %c
-  %smin2 = select i1 %c2, i32 %b, i32 %c
-  %c3 = icmp slt i32 %smin2, %a
-  %smax3 = select i1 %c3, i32 %smin2, i32 %a
+  %smax1 = call i32 @llvm.smax.i32(i32 %a, i32 %b)
+  %smin2 = call i32 @llvm.smin.i32(i32 %b, i32 %c)
+  %smax3 = call i32 @llvm.smin.i32(i32 %smin2, i32 %a)
   %res = add i32 %smax1, %smax3
   ret i32 %res
 }
diff --git a/llvm/test/Transforms/NaryReassociate/nary-smin.ll b/llvm/test/Transforms/NaryReassociate/nary-smin.ll
index a8e9fe8247c7a..c0416999681e5 100644
--- a/llvm/test/Transforms/NaryReassociate/nary-smin.ll
+++ b/llvm/test/Transforms/NaryReassociate/nary-smin.ll
@@ -7,18 +7,14 @@ declare i32 @llvm.smin.i32(i32 %a, i32 %b)
 ; m2 = smin(smin((b,c), a) -> m2 = smin(m1, c)
 define i32 @smin_test1(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @smin_test1(
-; CHECK-NEXT:    [[C1:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SMIN1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    [[SMIN1:%.*]] = call i32 @llvm.smin.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    [[SMIN3_NARY:%.*]] = call i32 @llvm.smin.i32(i32 [[SMIN1]], i32 [[C:%.*]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMIN1]], [[SMIN3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
-  %c1 = icmp slt i32 %a, %b
-  %smin1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp slt i32 %b, %c
-  %smin2 = select i1 %c2, i32 %b, i32 %c
-  %c3 = icmp slt i32 %smin2, %a
-  %smin3 = select i1 %c3, i32 %smin2, i32 %a
+  %smin1 = call i32 @llvm.smin.i32(i32 %a, i32 %b)
+  %smin2 = call i32 @llvm.smin.i32(i32 %b, i32 %c)
+  %smin3 = call i32 @llvm.smin.i32(i32 %smin2, i32 %a)
   %res = add i32 %smin1, %smin3
   ret i32 %res
 }
@@ -27,33 +23,14 @@ define i32 @smin_test1(i32 %a, i32 %b, i32 %c) {
 ; m2 = smin(b, (smin(a, c))) -> m2 = smin(m1, c)
 define i32 @smin_test2(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @smin_test2(
-; CHECK-NEXT:    [[C1:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SMIN1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[SMIN3_NARY:%.*]] = call i32 @llvm.smin.i32(i32 [[SMIN1]], i32 [[C:%.*]])
-; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMIN1]], [[SMIN3_NARY]]
-; CHECK-NEXT:    ret i32 [[RES]]
-;
-  %c1 = icmp slt i32 %a, %b
-  %smin1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp slt i32 %a, %c
-  %smin2 = select i1 %c2, i32 %a, i32 %c
-  %c3 = icmp slt i32 %b, %smin2
-  %smin3 = select i1 %c3, i32 %b, i32 %smin2
-  %res = add i32 %smin1, %smin3
-  ret i32 %res
-}
-
-; Same test as smin_test1 but uses @llvm.smin intrinsic
-define i32 @smin_test3(i32 %a, i32 %b, i32 %c) {
-; CHECK-LABEL: @smin_test3(
 ; CHECK-NEXT:    [[SMIN1:%.*]] = call i32 @llvm.smin.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    [[SMIN3_NARY:%.*]] = call i32 @llvm.smin.i32(i32 [[SMIN1]], i32 [[C:%.*]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMIN1]], [[SMIN3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %smin1 = call i32 @llvm.smin.i32(i32 %a, i32 %b)
-  %smin2 = call i32 @llvm.smin.i32(i32 %b, i32 %c)
-  %smin3 = call i32 @llvm.smin.i32(i32 %smin2, i32 %a)
+  %smin2 = call i32 @llvm.smin.i32(i32 %a, i32 %c)
+  %smin3 = call i32 @llvm.smin.i32(i32 %b, i32 %smin2)
   %res = add i32 %smin1, %smin3
   ret i32 %res
 }
@@ -62,18 +39,14 @@ define i32 @smin_test3(i32 %a, i32 %b, i32 %c) {
 ; m2 = smin(smin_or_eq((b,c), a) -> m2 = smin(m1, c)
 define i32 @umin_test4(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @umin_test4(
-; CHECK-NEXT:    [[C1:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SMIN1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    [[SMIN1:%.*]] = call i32 @llvm.smin.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    [[SMIN3_NARY:%.*]] = call i32 @llvm.smin.i32(i32 [[SMIN1]], i32 [[C:%.*]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMIN1]], [[SMIN3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
-  %c1 = icmp slt i32 %a, %b
-  %smin1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp sle i32 %b, %c
-  %smin_or_eq2 = select i1 %c2, i32 %b, i32 %c
-  %c3 = icmp slt i32 %smin_or_eq2, %a
-  %smin3 = select i1 %c3, i32 %smin_or_eq2, i32 %a
+  %smin1 = call i32 @llvm.smin.i32(i32 %a, i32 %b)
+  %smin_or_eq2 = call i32 @llvm.smin.i32(i32 %b, i32 %c)
+  %smin3 = call i32 @llvm.smin.i32(i32 %smin_or_eq2, i32 %a)
   %res = add i32 %smin1, %smin3
   ret i32 %res
 }
@@ -82,18 +55,14 @@ define i32 @umin_test4(i32 %a, i32 %b, i32 %c) {
 ; m2 = smin_or_eq(smin((b,c), a) -> m2 = smin(m1, c)
 define i32 @smin_test5(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @smin_test5(
-; CHECK-NEXT:    [[C1:%.*]] = icmp sle i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SMIN_OR_EQ1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    [[SMIN_OR_EQ1:%.*]] = call i32 @llvm.smin.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    [[SMIN_OR_EQ3_NARY:%.*]] = call i32 @llvm.smin.i32(i32 [[SMIN_OR_EQ1]], i32 [[C:%.*]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMIN_OR_EQ1]], [[SMIN_OR_EQ3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
-  %c1 = icmp sle i32 %a, %b
-  %smin_or_eq1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp slt i32 %b, %c
-  %smin2 = select i1 %c2, i32 %b, i32 %c
-  %c3 = icmp sle i32 %smin2, %a
-  %smin_or_eq3 = select i1 %c3, i32 %smin2, i32 %a
+  %smin_or_eq1 = call i32 @llvm.smin.i32(i32 %a, i32 %b)
+  %smin2 = call i32 @llvm.smin.i32(i32 %b, i32 %c)
+  %smin_or_eq3 = call i32 @llvm.smin.i32(i32 %smin2, i32 %a)
   %res = add i32 %smin_or_eq1, %smin_or_eq3
   ret i32 %res
 }
@@ -102,21 +71,15 @@ define i32 @smin_test5(i32 %a, i32 %b, i32 %c) {
 ; m2 = smin(umin((b,c), a) ; check that signed and unsigned mins are not mixed
 define i32 @smin_test6(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @smin_test6(
-; CHECK-NEXT:    [[C1:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SMIN1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[C2:%.*]] = icmp ult i32 [[B]], [[C:%.*]]
-; CHECK-NEXT:    [[UMIN2:%.*]] = select i1 [[C2]], i32 [[B]], i32 [[C]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp slt i32 [[UMIN2]], [[A]]
-; CHECK-NEXT:    [[SMIN3:%.*]] = select i1 [[C3]], i32 [[UMIN2]], i32 [[A]]
+; CHECK-NEXT:    [[SMIN1:%.*]] = call i32 @llvm.smin.i32(i32 [[A:%.*]], i32 [[B:%.*]])
+; CHECK-NEXT:    [[UMIN2:%.*]] = call i32 @llvm.umin.i32(i32 [[B]], i32 [[C:%.*]])
+; CHECK-NEXT:    [[SMIN3:%.*]] = call i32 @llvm.smin.i32(i32 [[UMIN2]], i32 [[A]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMIN1]], [[SMIN3]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
-  %c1 = icmp slt i32 %a, %b
-  %smin1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp ult i32 %b, %c
-  %umin2 = select i1 %c2, i32 %b, i32 %c
-  %c3 = icmp slt i32 %umin2, %a
-  %smin3 = select i1 %c3, i32 %umin2, i32 %a
+  %smin1 = call i32 @llvm.smin.i32(i32 %a, i32 %b)
+  %umin2 = call i32 @llvm.umin.i32(i32 %b, i32 %c)
+  %smin3 = call i32 @llvm.smin.i32(i32 %umin2, i32 %a)
   %res = add i32 %smin1, %smin3
   ret i32 %res
 }
@@ -125,21 +88,15 @@ define i32 @smin_test6(i32 %a, i32 %b, i32 %c) {
 ; m2 = smin(smax((b,c), a) ; check that min and max are not mixed
 define i32 @smin_test7(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @smin_test7(
-; CHECK-NEXT:    [[C1:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SMIN1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[C2:%.*]] = icmp sgt i32 [[B]], [[C:%.*]]
-; CHECK-NEXT:    [[SMAX2:%.*]] = select i1 [[C2]], i32 [[B]], i32 [[C]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp slt i32 [[SMAX2]], [[A]]
-; CHECK-NEXT:    [[SMIN3:%.*]] = select i1 [[C3]], i32 [[SMAX2]], i32 [[A]]
+; CHECK-NEXT:    [[SMIN1:%.*]] = call i32 @llvm.smin.i32(i32 [[A:%.*]], i32 [[B:%.*]])
+; CHECK-NEXT:    [[SMAX2:%.*]] = call i32 @llvm.smax.i32(i32 [[B]], i32 [[C:%.*]])
+; CHECK-NEXT:    [[SMIN3:%.*]] = call i32 @llvm.smin.i32(i32 [[SMAX2]], i32 [[A]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMIN1]], [[SMIN3]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
-  %c1 = icmp slt i32 %a, %b
-  %smin1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp sgt i32 %b, %c
-  %smax2 = select i1 %c2, i32 %b, i32 %c
-  %c3 = icmp slt i32 %smax2, %a
-  %smin3 = select i1 %c3, i32 %smax2, i32 %a
+  %smin1 = call i32 @llvm.smin.i32(i32 %a, i32 %b)
+  %smax2 = call i32 @llvm.smax.i32(i32 %b, i32 %c)
+  %smin3 = call i32 @llvm.smin.i32(i32 %smax2, i32 %a)
   %res = add i32 %smin1, %smin3
   ret i32 %res
 }
diff --git a/llvm/test/Transforms/NaryReassociate/nary-umax.ll b/llvm/test/Transforms/NaryReassociate/nary-umax.ll
index b391fc4ba3ebb..08c9875e05278 100644
--- a/llvm/test/Transforms/NaryReassociate/nary-umax.ll
+++ b/llvm/test/Transforms/NaryReassociate/nary-umax.ll
@@ -7,18 +7,14 @@ declare i32 @llvm.umax.i32(i32 %a, i32 %b)
 ; m2 = umax(umax((b,c), a) -> m2 = umax(m1, c)
 define i32 @umax_test1(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @umax_test1(
-; CHECK-NEXT:    [[C1:%.*]] = icmp ugt i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[UMAX1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    [[UMAX1:%.*]] = call i32 @llvm.umax.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    [[UMAX3_NARY:%.*]] = call i32 @llvm.umax.i32(i32 [[UMAX1]], i32 [[C:%.*]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMAX1]], [[UMAX3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
-  %c1 = icmp ugt i32 %a, %b
-  %umax1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp ugt i32 %b, %c
-  %umax2 = select i1 %c2, i32 %b, i32 %c
-  %c3 = icmp ugt i32 %umax2, %a
-  %umax3 = select i1 %c3, i32 %umax2, i32 %a
+  %umax1 = call i32 @llvm.umax.i32(i32 %a, i32 %b)
+  %umax2 = call i32 @llvm.umax.i32(i32 %b, i32 %c)
+  %umax3 = call i32 @llvm.umax.i32(i32 %umax2, i32 %a)
   %res = add i32 %umax1, %umax3
   ret i32 %res
 }
@@ -27,33 +23,14 @@ define i32 @umax_test1(i32 %a, i32 %b, i32 %c) {
 ; m2 = umax(b, (umax(a, c))) -> m2 = umax(m1, c)
 define i32 @umax_test2(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @umax_test2(
-; CHECK-NEXT:    [[C1:%.*]] = icmp ugt i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[UMAX1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[UMAX3_NARY:%.*]] = call i32 @llvm.umax.i32(i32 [[UMAX1]], i32 [[C:%.*]])
-; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMAX1]], [[UMAX3_NARY]]
-; CHECK-NEXT:    ret i32 [[RES]]
-;
-  %c1 = icmp ugt i32 %a, %b
-  %umax1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp ugt i32 %a, %c
-  %umax2 = select i1 %c2, i32 %a, i32 %c
-  %c3 = icmp ugt i32 %b, %umax2
-  %umax3 = select i1 %c3, i32 %b, i32 %umax2
-  %res = add i32 %umax1, %umax3
-  ret i32 %res
-}
-
-; Same test as umax_test1 but uses @llvm.umax intrinsic
-define i32 @umax_test3(i32 %a, i32 %b, i32 %c) {
-; CHECK-LABEL: @umax_test3(
 ; CHECK-NEXT:    [[UMAX1:%.*]] = call i32 @llvm.umax.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    [[UMAX3_NARY:%.*]] = call i32 @llvm.umax.i32(i32 [[UMAX1]], i32 [[C:%.*]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMAX1]], [[UMAX3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %umax1 = call i32 @llvm.umax.i32(i32 %a, i32 %b)
-  %umax2 = call i32 @llvm.umax.i32(i32 %b, i32 %c)
-  %umax3 = call i32 @llvm.umax.i32(i32 %umax2, i32 %a)
+  %umax2 = call i32 @llvm.umax.i32(i32 %a, i32 %c)
+  %umax3 = call i32 @llvm.umax.i32(i32 %b, i32 %umax2)
   %res = add i32 %umax1, %umax3
   ret i32 %res
 }
@@ -62,18 +39,14 @@ define i32 @umax_test3(i32 %a, i32 %b, i32 %c) {
 ; m2 = umax(umax_or_eq((b,c), a) -> m2 = umax(m1, c)
 define i32 @umax_test4(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @umax_test4(
-; CHECK-NEXT:    [[C1:%.*]] = icmp ugt i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[UMAX1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    [[UMAX1:%.*]] = call i32 @llvm.umax.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    [[UMAX3_NARY:%.*]] = call i32 @llvm.umax.i32(i32 [[UMAX1]], i32 [[C:%.*]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMAX1]], [[UMAX3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
-  %c1 = icmp ugt i32 %a, %b
-  %umax1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp uge i32 %b, %c
-  %umax_or_eq2 = select i1 %c2, i32 %b, i32 %c
-  %c3 = icmp ugt i32 %umax_or_eq2, %a
-  %umax3 = select i1 %c3, i32 %umax_or_eq2, i32 %a
+  %umax1 = call i32 @llvm.umax.i32(i32 %a, i32 %b)
+  %umax_or_eq2 = call i32 @llvm.umax.i32(i32 %b, i32 %c)
+  %umax3 = call i32 @llvm.umax.i32(i32 %umax_or_eq2, i32 %a)
   %res = add i32 %umax1, %umax3
   ret i32 %res
 }
@@ -82,18 +55,14 @@ define i32 @umax_test4(i32 %a, i32 %b, i32 %c) {
 ; m2 = umax_or_eq(umax((b,c), a) -> m2 = umax(m1, c)
 define i32 @umax_test5(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @umax_test5(
-; CHECK-NEXT:    [[C1:%.*]] = icmp uge i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[UMAX_OR_EQ1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    [[UMAX_OR_EQ1:%.*]] = call i32 @llvm.umax.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    [[UMAX_OR_EQ3_NARY:%.*]] = call i32 @llvm.umax.i32(i32 [[UMAX_OR_EQ1]], i32 [[C:%.*]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMAX_OR_EQ1]], [[UMAX_OR_EQ3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
-  %c1 = icmp uge i32 %a, %b
-  %umax_or_eq1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp ugt i32 %b, %c
-  %umax2 = select i1 %c2, i32 %b, i32 %c
-  %c3 = icmp uge i32 %umax2, %a
-  %umax_or_eq3 = select i1 %c3, i32 %umax2, i32 %a
+  %umax_or_eq1 = call i32 @llvm.umax.i32(i32 %a, i32 %b)
+  %umax2 = call i32 @llvm.umax.i32(i32 %b, i32 %c)
+  %umax_or_eq3 = call i32 @llvm.umax.i32(i32 %umax2, i32 %a)
   %res = add i32 %umax_or_eq1, %umax_or_eq3
   ret i32 %res
 }
@@ -102,21 +71,15 @@ define i32 @umax_test5(i32 %a, i32 %b, i32 %c) {
 ; m2 = umax(smax((b,c), a) ; check that signed and unsigned maxs are not mixed
 define i32 @umax_test6(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @umax_test6(
-; CHECK-NEXT:    [[C1:%.*]] = icmp ugt i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[UMAX1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[C2:%.*]] = icmp sgt i32 [[B]], [[C:%.*]]
-; CHECK-NEXT:    [[SMAX2:%.*]] = select i1 [[C2]], i32 [[B]], i32 [[C]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp ugt i32 [[SMAX2]], [[A]]
-; CHECK-NEXT:    [[UMAX3:%.*]] = select i1 [[C3]], i32 [[SMAX2]], i32 [[A]]
+; CHECK-NEXT:    [[UMAX1:%.*]] = call i32 @llvm.umax.i32(i32 [[A:%.*]], i32 [[B:%.*]])
+; CHECK-NEXT:    [[SMAX2:%.*]] = call i32 @llvm.smax.i32(i32 [[B]], i32 [[C:%.*]])
+; CHECK-NEXT:    [[UMAX3:%.*]] = call i32 @llvm.umax.i32(i32 [[SMAX2]], i32 [[A]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMAX1]], [[UMAX3]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
-  %c1 = icmp ugt i32 %a, %b
-  %umax1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp sgt i32 %b, %c
-  %smax2 = select i1 %c2, i32 %b, i32 %c
-  %c3 = icmp ugt i32 %smax2, %a
-  %umax3 = select i1 %c3, i32 %smax2, i32 %a
+  %umax1 = call i32 @llvm.umax.i32(i32 %a, i32 %b)
+  %smax2 = call i32 @llvm.smax.i32(i32 %b, i32 %c)
+  %umax3 = call i32 @llvm.umax.i32(i32 %smax2, i32 %a)
   %res = add i32 %umax1, %umax3
   ret i32 %res
 }
@@ -125,21 +88,15 @@ define i32 @umax_test6(i32 %a, i32 %b, i32 %c) {
 ; m2 = umax(umin((b,c), a) ; check that max and min are not mixed
 define i32 @umax_test7(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @umax_test7(
-; CHECK-NEXT:    [[C1:%.*]] = icmp ugt i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[UMAX1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[C2:%.*]] = icmp ult i32 [[B]], [[C:%.*]]
-; CHECK-NEXT:    [[UMAX2:%.*]] = select i1 [[C2]], i32 [[B]], i32 [[C]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp ugt i32 [[UMAX2]], [[A]]
-; CHECK-NEXT:    [[UMAX3:%.*]] = select i1 [[C3]], i32 [[UMAX2]], i32 [[A]]
+; CHECK-NEXT:    [[UMAX1:%.*]] = call i32 @llvm.umax.i32(i32 [[A:%.*]], i32 [[B:%.*]])
+; CHECK-NEXT:    [[UMAX2:%.*]] = call i32 @llvm.umin.i32(i32 [[B]], i32 [[C:%.*]])
+; CHECK-NEXT:    [[UMAX3:%.*]] = call i32 @llvm.umax.i32(i32 [[UMAX2]], i32 [[A]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMAX1]], [[UMAX3]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
-  %c1 = icmp ugt i32 %a, %b
-  %umax1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp ult i32 %b, %c
-  %umax2 = select i1 %c2, i32 %b, i32 %c
-  %c3 = icmp ugt i32 %umax2, %a
-  %umax3 = select i1 %c3, i32 %umax2, i32 %a
+  %umax1 = call i32 @llvm.umax.i32(i32 %a, i32 %b)
+  %umax2 = call i32 @llvm.umin.i32(i32 %b, i32 %c)
+  %umax3 = call i32 @llvm.umax.i32(i32 %umax2, i32 %a)
   %res = add i32 %umax1, %umax3
   ret i32 %res
 }
diff --git a/llvm/test/Transforms/NaryReassociate/nary-umin.ll b/llvm/test/Transforms/NaryReassociate/nary-umin.ll
index 58fdcc3f47a5a..b0be4ea801f63 100644
--- a/llvm/test/Transforms/NaryReassociate/nary-umin.ll
+++ b/llvm/test/Transforms/NaryReassociate/nary-umin.ll
@@ -7,18 +7,14 @@ declare i32 @llvm.umin.i32(i32 %a, i32 %b)
 ; m2 = umin(umin((b,c), a) -> m2 = umin(m1, c)
 define i32 @umin_test1(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @umin_test1(
-; CHECK-NEXT:    [[C1:%.*]] = icmp ult i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[UMIN1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    [[UMIN1:%.*]] = call i32 @llvm.umin.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    [[UMIN3_NARY:%.*]] = call i32 @llvm.umin.i32(i32 [[UMIN1]], i32 [[C:%.*]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMIN1]], [[UMIN3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
-  %c1 = icmp ult i32 %a, %b
-  %umin1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp ult i32 %b, %c
-  %umin2 = select i1 %c2, i32 %b, i32 %c
-  %c3 = icmp ult i32 %umin2, %a
-  %umin3 = select i1 %c3, i32 %umin2, i32 %a
+  %umin1 = call i32 @llvm.umin.i32(i32 %a, i32 %b)
+  %umin2 = call i32 @llvm.umin.i32(i32 %b, i32 %c)
+  %umin3 = call i32 @llvm.umin.i32(i32 %umin2, i32 %a)
   %res = add i32 %umin1, %umin3
   ret i32 %res
 }
@@ -27,33 +23,14 @@ define i32 @umin_test1(i32 %a, i32 %b, i32 %c) {
 ; m2 = umin(b, (umin(a, c))) -> m2 = umin(m1, c)
 define i32 @umin_test2(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @umin_test2(
-; CHECK-NEXT:    [[C1:%.*]] = icmp ult i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[UMIN1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[UMIN3_NARY:%.*]] = call i32 @llvm.umin.i32(i32 [[UMIN1]], i32 [[C:%.*]])
-; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMIN1]], [[UMIN3_NARY]]
-; CHECK-NEXT:    ret i32 [[RES]]
-;
-  %c1 = icmp ult i32 %a, %b
-  %umin1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp ult i32 %a, %c
-  %umin2 = select i1 %c2, i32 %a, i32 %c
-  %c3 = icmp ult i32 %b, %umin2
-  %umin3 = select i1 %c3, i32 %b, i32 %umin2
-  %res = add i32 %umin1, %umin3
-  ret i32 %res
-}
-
-; Same test as umin_test1 but uses @llvm.umin intrinsic
-define i32 @umin_test3(i32 %a, i32 %b, i32 %c) {
-; CHECK-LABEL: @umin_test3(
 ; CHECK-NEXT:    [[UMIN1:%.*]] = call i32 @llvm.umin.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    [[UMIN3_NARY:%.*]] = call i32 @llvm.umin.i32(i32 [[UMIN1]], i32 [[C:%.*]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMIN1]], [[UMIN3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %umin1 = call i32 @llvm.umin.i32(i32 %a, i32 %b)
-  %umin2 = call i32 @llvm.umin.i32(i32 %b, i32 %c)
-  %umin3 = call i32 @llvm.umin.i32(i32 %umin2, i32 %a)
+  %umin2 = call i32 @llvm.umin.i32(i32 %a, i32 %c)
+  %umin3 = call i32 @llvm.umin.i32(i32 %b, i32 %umin2)
   %res = add i32 %umin1, %umin3
   ret i32 %res
 }
@@ -62,18 +39,14 @@ define i32 @umin_test3(i32 %a, i32 %b, i32 %c) {
 ; m2 = umin(umin_or_eq((b,c), a) -> m2 = umin(m1, c)
 define i32 @umin_test4(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @umin_test4(
-; CHECK-NEXT:    [[C1:%.*]] = icmp ult i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[UMIN1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    [[UMIN1:%.*]] = call i32 @llvm.umin.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    [[UMIN3_NARY:%.*]] = call i32 @llvm.umin.i32(i32 [[UMIN1]], i32 [[C:%.*]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMIN1]], [[UMIN3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
-  %c1 = icmp ult i32 %a, %b
-  %umin1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp ule i32 %b, %c
-  %umin_or_eq2 = select i1 %c2, i32 %b, i32 %c
-  %c3 = icmp ult i32 %umin_or_eq2, %a
-  %umin3 = select i1 %c3, i32 %umin_or_eq2, i32 %a
+  %umin1 = call i32 @llvm.umin.i32(i32 %a, i32 %b)
+  %umin_or_eq2 = call i32 @llvm.umin.i32(i32 %b, i32 %c)
+  %umin3 = call i32 @llvm.umin.i32(i32 %umin_or_eq2, i32 %a)
   %res = add i32 %umin1, %umin3
   ret i32 %res
 }
@@ -82,18 +55,14 @@ define i32 @umin_test4(i32 %a, i32 %b, i32 %c) {
 ; m2 = umin_or_eq(umin((b,c), a) -> m2 = umin(m1, c)
 define i32 @umin_test5(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @umin_test5(
-; CHECK-NEXT:    [[C1:%.*]] = icmp ule i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[UMIN_OR_EQ1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    [[UMIN_OR_EQ1:%.*]] = call i32 @llvm.umin.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    [[UMIN_OR_EQ3_NARY:%.*]] = call i32 @llvm.umin.i32(i32 [[UMIN_OR_EQ1]], i32 [[C:%.*]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMIN_OR_EQ1]], [[UMIN_OR_EQ3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
-  %c1 = icmp ule i32 %a, %b
-  %umin_or_eq1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp ult i32 %b, %c
-  %umin2 = select i1 %c2, i32 %b, i32 %c
-  %c3 = icmp ule i32 %umin2, %a
-  %umin_or_eq3 = select i1 %c3, i32 %umin2, i32 %a
+  %umin_or_eq1 = call i32 @llvm.umin.i32(i32 %a, i32 %b)
+  %umin2 = call i32 @llvm.umin.i32(i32 %b, i32 %c)
+  %umin_or_eq3 = call i32 @llvm.umin.i32(i32 %umin2, i32 %a)
   %res = add i32 %umin_or_eq1, %umin_or_eq3
   ret i32 %res
 }
@@ -102,21 +71,15 @@ define i32 @umin_test5(i32 %a, i32 %b, i32 %c) {
 ; m2 = umin(smin((b,c), a) ; check that signed and unsigned mins are not mixed
 define i32 @umin_test6(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @umin_test6(
-; CHECK-NEXT:    [[C1:%.*]] = icmp ult i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[UMIN1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[C2:%.*]] = icmp slt i32 [[B]], [[C:%.*]]
-; CHECK-NEXT:    [[SMIN2:%.*]] = select i1 [[C2]], i32 [[B]], i32 [[C]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp ult i32 [[SMIN2]], [[A]]
-; CHECK-NEXT:    [[UMIN3:%.*]] = select i1 [[C3]], i32 [[SMIN2]], i32 [[A]]
+; CHECK-NEXT:    [[UMIN1:%.*]] = call i32 @llvm.umin.i32(i32 [[A:%.*]], i32 [[B:%.*]])
+; CHECK-NEXT:    [[SMIN2:%.*]] = call i32 @llvm.smin.i32(i32 [[B]], i32 [[C:%.*]])
+; CHECK-NEXT:    [[UMIN3:%.*]] = call i32 @llvm.umin.i32(i32 [[SMIN2]], i32 [[A]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMIN1]], [[UMIN3]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
-  %c1 = icmp ult i32 %a, %b
-  %umin1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp slt i32 %b, %c
-  %smin2 = select i1 %c2, i32 %b, i32 %c
-  %c3 = icmp ult i32 %smin2, %a
-  %umin3 = select i1 %c3, i32 %smin2, i32 %a
+  %umin1 = call i32 @llvm.umin.i32(i32 %a, i32 %b)
+  %smin2 = call i32 @llvm.smin.i32(i32 %b, i32 %c)
+  %umin3 = call i32 @llvm.umin.i32(i32 %smin2, i32 %a)
   %res = add i32 %umin1, %umin3
   ret i32 %res
 }
@@ -125,21 +88,15 @@ define i32 @umin_test6(i32 %a, i32 %b, i32 %c) {
 ; m2 = umin(umax((b,c), a) ; check that min and max are not mixed
 define i32 @umin_test7(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @umin_test7(
-; CHECK-NEXT:    [[C1:%.*]] = icmp ult i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[UMIN1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[C2:%.*]] = icmp ugt i32 [[B]], [[C:%.*]]
-; CHECK-NEXT:    [[UMAX2:%.*]] = select i1 [[C2]], i32 [[B]], i32 [[C]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp ult i32 [[UMAX2]], [[A]]
-; CHECK-NEXT:    [[UMIN3:%.*]] = select i1 [[C3]], i32 [[UMAX2]], i32 [[A]]
+; CHECK-NEXT:    [[UMIN1:%.*]] = call i32 @llvm.umin.i32(i32 [[A:%.*]], i32 [[B:%.*]])
+; CHECK-NEXT:    [[UMAX2:%.*]] = call i32 @llvm.umax.i32(i32 [[B]], i32 [[C:%.*]])
+; CHECK-NEXT:    [[UMIN3:%.*]] = call i32 @llvm.umin.i32(i32 [[UMAX2]], i32 [[A]])
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMIN1]], [[UMIN3]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
-  %c1 = icmp ult i32 %a, %b
-  %umin1 = select i1 %c1, i32 %a, i32 %b
-  %c2 = icmp ugt i32 %b, %c
-  %umax2 = select i1 %c2, i32 %b, i32 %c
-  %c3 = icmp ult i32 %umax2, %a
-  %umin3 = select i1 %c3, i32 %umax2, i32 %a
+  %umin1 = call i32 @llvm.umin.i32(i32 %a, i32 %b)
+  %umax2 = call i32 @llvm.umax.i32(i32 %b, i32 %c)
+  %umin3 = call i32 @llvm.umin.i32(i32 %umax2, i32 %a)
   %res = add i32 %umin1, %umin3
   ret i32 %res
 }
diff --git a/llvm/test/Transforms/SCCP/noipa.ll b/llvm/test/Transforms/SCCP/noipa.ll
new file mode 100644
index 0000000000000..f4b29e3ee4096
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/noipa.ll
@@ -0,0 +1,25 @@
+; RUN: opt -passes=ipsccp -S < %s | FileCheck %s
+
+; IPSCCP constant-propagating through arguments of a `noipa` function.
+; If `noipa` is working, IPSCCP should NOT propagate the constant 42 into @foo,
+; because analyzing calls to @foo to determine argument values is IPA.
+
+; CHECK:      define internal i32 @foo(i32 %x)
+; CHECK-NEXT:   ret i32 poison
+define internal i32 @foo(i32 %x) noinline {
+  ret i32 %x
+}
+
+; CHECK:      define internal i32 @foo_noipa(i32 %x)
+; CHECK-NEXT:   ret i32 %x
+define internal i32 @foo_noipa(i32 %x) noipa noinline {
+  ret i32 %x
+}
+
+define i32 @bar() {
+  %r = call i32 @foo(i32 42)
+  %s = call i32 @foo_noipa(i32 42)
+  ; CHECK: %res = add i32 42, %s
+  %res = add i32 %r, %s
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/horizontal-store.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/horizontal-store.ll
index 72fb03e60a297..c6dc69e69dd31 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/horizontal-store.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/horizontal-store.ll
@@ -15,40 +15,31 @@
 ; Tests whether the min/max reduction pattern is vectorized if SLP starts at the store.
 define i32 @smaxv6() {
 ; GFX9-LABEL: @smaxv6(
-; GFX9-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @arr, align 16
-; GFX9-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; GFX9-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; GFX9-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @arr, align 16
+; GFX9-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
+; GFX9-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
 ; GFX9-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
-; GFX9-NEXT:    [[SELECT1:%.*]] = select i1 [[CMP1]], i32 [[TMP2]], i32 [[TMP3]]
-; GFX9-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 2), align 8
-; GFX9-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP4]])
-; GFX9-NEXT:    [[OP_RDX:%.*]] = icmp sgt i32 [[TMP5]], [[SELECT1]]
-; GFX9-NEXT:    [[OP_RDX1:%.*]] = select i1 [[OP_RDX]], i32 [[TMP5]], i32 [[SELECT1]]
+; GFX9-NEXT:    [[LOAD5:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 16), align 16
+; GFX9-NEXT:    [[LOAD6:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 20), align 4
+; GFX9-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP1]])
+; GFX9-NEXT:    [[TMP5:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP4]], i32 [[LOAD5]])
+; GFX9-NEXT:    [[TMP6:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP5]], i32 [[LOAD6]])
 ; GFX9-NEXT:    [[STORE_SELECT:%.*]] = select i1 [[CMP1]], i32 3, i32 4
 ; GFX9-NEXT:    store i32 [[STORE_SELECT]], ptr @var, align 8
-; GFX9-NEXT:    ret i32 [[OP_RDX1]]
+; GFX9-NEXT:    ret i32 [[TMP6]]
 ;
   %load1 = load i32, ptr @arr, align 16
   %load2 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 1), align 4
   %cmp1 = icmp sgt i32 %load1, %load2
-  %select1 = select i1 %cmp1, i32 %load1, i32 %load2
-
-  %load3 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 2), align 8
-  %cmp2 = icmp sgt i32 %select1, %load3
-  %select2 = select i1 %cmp2, i32 %select1, i32 %load3
-
-  %load4 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 3), align 4
-  %cmp3 = icmp sgt i32 %select2, %load4
-  %select3 = select i1 %cmp3, i32 %select2, i32 %load4
-
-  %load5 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 4), align 16
-  %cmp4 = icmp sgt i32 %select3, %load5
-  %select4 = select i1 %cmp4, i32 %select3, i32 %load5
-
-  %load6 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 5), align 4
-  %cmp5 = icmp sgt i32 %select4, %load6
-  %select5 = select i1 %cmp5, i32 %select4, i32 %load6
-
+  %select1 = call i32 @llvm.smax.i32(i32 %load1, i32 %load2)
+  %load3 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 8), align 8
+  %select2 = call i32 @llvm.smax.i32(i32 %select1, i32 %load3)
+  %load4 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 12), align 4
+  %select3 = call i32 @llvm.smax.i32(i32 %select2, i32 %load4)
+  %load5 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 16), align 16
+  %select4 = call i32 @llvm.smax.i32(i32 %select3, i32 %load5)
+  %load6 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 20), align 4
+  %select5 = call i32 @llvm.smax.i32(i32 %select4, i32 %load6)
   %store-select = select i1 %cmp1, i32 3, i32 4
   store i32 %store-select, ptr @var, align 8
   ret i32 %select5
@@ -56,40 +47,31 @@ define i32 @smaxv6() {
 
 define i64 @sminv6() {
 ; GFX9-LABEL: @sminv6(
-; GFX9-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @arr64, align 16
-; GFX9-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
-; GFX9-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; GFX9-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @arr64, align 16
+; GFX9-NEXT:    [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i32 0
+; GFX9-NEXT:    [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i32 1
 ; GFX9-NEXT:    [[CMP1:%.*]] = icmp slt i64 [[TMP2]], [[TMP3]]
-; GFX9-NEXT:    [[SELECT1:%.*]] = select i1 [[CMP1]], i64 [[TMP2]], i64 [[TMP3]]
-; GFX9-NEXT:    [[TMP4:%.*]] = load <4 x i64>, ptr getelementptr inbounds ([32 x i64], ptr @arr64, i64 0, i64 2), align 16
-; GFX9-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> [[TMP4]])
-; GFX9-NEXT:    [[OP_RDX:%.*]] = icmp slt i64 [[TMP5]], [[SELECT1]]
-; GFX9-NEXT:    [[OP_RDX1:%.*]] = select i1 [[OP_RDX]], i64 [[TMP5]], i64 [[SELECT1]]
+; GFX9-NEXT:    [[LOAD5:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @arr64, i64 32), align 16
+; GFX9-NEXT:    [[LOAD6:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @arr64, i64 40), align 8
+; GFX9-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> [[TMP1]])
+; GFX9-NEXT:    [[TMP5:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP4]], i64 [[LOAD5]])
+; GFX9-NEXT:    [[TMP6:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP5]], i64 [[LOAD6]])
 ; GFX9-NEXT:    [[STORE_SELECT:%.*]] = select i1 [[CMP1]], i64 3, i64 4
 ; GFX9-NEXT:    store i64 [[STORE_SELECT]], ptr @var64, align 8
-; GFX9-NEXT:    ret i64 [[OP_RDX1]]
+; GFX9-NEXT:    ret i64 [[TMP6]]
 ;
   %load1 = load i64, ptr @arr64, align 16
   %load2 = load i64, ptr getelementptr inbounds ([32 x i64], ptr @arr64, i64 0, i64 1), align 8
   %cmp1 = icmp slt i64 %load1, %load2
-  %select1 = select i1 %cmp1, i64 %load1, i64 %load2
-
-  %load3 = load i64, ptr getelementptr inbounds ([32 x i64], ptr @arr64, i64 0, i64 2), align 16
-  %cmp2 = icmp slt i64 %select1, %load3
-  %select2 = select i1 %cmp2, i64 %select1, i64 %load3
-
-  %load4 = load i64, ptr getelementptr inbounds ([32 x i64], ptr @arr64, i64 0, i64 3), align 8
-  %cmp3 = icmp slt i64 %select2, %load4
-  %select3 = select i1 %cmp3, i64 %select2, i64 %load4
-
-  %load5 = load i64, ptr getelementptr inbounds ([32 x i64], ptr @arr64, i64 0, i64 4), align 16
-  %cmp4 = icmp slt i64 %select3, %load5
-  %select4 = select i1 %cmp4, i64 %select3, i64 %load5
-
-  %load6 = load i64, ptr getelementptr inbounds ([32 x i64], ptr @arr64, i64 0, i64 5), align 8
-  %cmp5 = icmp slt i64 %select4, %load6
-  %select5 = select i1 %cmp5, i64 %select4, i64 %load6
-
+  %select1 = call i64 @llvm.smin.i64(i64 %load1, i64 %load2)
+  %load3 = load i64, ptr getelementptr inbounds nuw (i8, ptr @arr64, i64 16), align 16
+  %select2 = call i64 @llvm.smin.i64(i64 %select1, i64 %load3)
+  %load4 = load i64, ptr getelementptr inbounds nuw (i8, ptr @arr64, i64 24), align 8
+  %select3 = call i64 @llvm.smin.i64(i64 %select2, i64 %load4)
+  %load5 = load i64, ptr getelementptr inbounds nuw (i8, ptr @arr64, i64 32), align 16
+  %select4 = call i64 @llvm.smin.i64(i64 %select3, i64 %load5)
+  %load6 = load i64, ptr getelementptr inbounds nuw (i8, ptr @arr64, i64 40), align 8
+  %select5 = call i64 @llvm.smin.i64(i64 %select4, i64 %load6)
   %store-select = select i1 %cmp1, i64 3, i64 4
   store i64 %store-select, ptr @var64, align 8
   ret i64 %select5
@@ -197,43 +179,33 @@ define double @dminv6() {
   ret double %select5
 }
 
-define i32 @smax_wdiff_valuenum(i32, i32 %v1) {
+define i32 @smax_wdiff_valuenum(i32 %0, i32 %v1) {
 ; GFX9-LABEL: @smax_wdiff_valuenum(
 ; GFX9-NEXT:    [[VLOAD:%.*]] = load <2 x i32>, ptr @arr, align 16
 ; GFX9-NEXT:    [[ELT1:%.*]] = extractelement <2 x i32> [[VLOAD]], i32 0
 ; GFX9-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[ELT1]], [[V1:%.*]]
 ; GFX9-NEXT:    [[EX0:%.*]] = extractelement <2 x i32> [[VLOAD]], i32 0
 ; GFX9-NEXT:    [[SELECT1:%.*]] = select i1 [[CMP1]], i32 [[EX0]], i32 [[V1]]
-; GFX9-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 2), align 8
+; GFX9-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 8), align 8
 ; GFX9-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP2]])
-; GFX9-NEXT:    [[OP_RDX:%.*]] = icmp sgt i32 [[TMP3]], [[SELECT1]]
-; GFX9-NEXT:    [[OP_RDX1:%.*]] = select i1 [[OP_RDX]], i32 [[TMP3]], i32 [[SELECT1]]
+; GFX9-NEXT:    [[TMP4:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP3]], i32 [[SELECT1]])
 ; GFX9-NEXT:    [[STOREVAL:%.*]] = select i1 [[CMP1]], i32 3, i32 4
 ; GFX9-NEXT:    store i32 [[STOREVAL]], ptr @var, align 8
-; GFX9-NEXT:    ret i32 [[OP_RDX1]]
+; GFX9-NEXT:    ret i32 [[TMP4]]
 ;
   %vload = load <2 x i32>, ptr @arr, align 16
   %elt1 = extractelement <2 x i32> %vload, i32 0
   %cmp1 = icmp sgt i32 %elt1, %v1
   %ex0 = extractelement <2 x i32> %vload, i32 0
   %select1 = select i1 %cmp1, i32 %ex0, i32 %v1
-
-  %load3 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 2), align 8
-  %cmp2 = icmp sgt i32 %select1, %load3
-  %select2 = select i1 %cmp2, i32 %select1, i32 %load3
-
-  %load4 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 3), align 4
-  %cmp3 = icmp sgt i32 %select2, %load4
-  %select3 = select i1 %cmp3, i32 %select2, i32 %load4
-
-  %load5 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 4), align 16
-  %cmp4 = icmp sgt i32 %select3, %load5
-  %select4 = select i1 %cmp4, i32 %select3, i32 %load5
-
-  %load6 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 5), align 4
-  %cmp5 = icmp sgt i32 %select4, %load6
-  %select5 = select i1 %cmp5, i32 %select4, i32 %load6
-
+  %load3 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 8), align 8
+  %select2 = call i32 @llvm.smax.i32(i32 %select1, i32 %load3)
+  %load4 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 12), align 4
+  %select3 = call i32 @llvm.smax.i32(i32 %select2, i32 %load4)
+  %load5 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 16), align 16
+  %select4 = call i32 @llvm.smax.i32(i32 %select3, i32 %load5)
+  %load6 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 20), align 4
+  %select5 = call i32 @llvm.smax.i32(i32 %select4, i32 %load6)
   %storeval = select i1 %cmp1, i32 3, i32 4
   store i32 %storeval, ptr @var, align 8
   ret i32 %select5
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/reduction.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/reduction.ll
index 88954cf4cb46d..f934e432cdd63 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/reduction.ll
@@ -270,12 +270,9 @@ define i16 @reduction_umin_v4i16(<4 x i16> %vec4) {
 ; VI-NEXT:    [[ELT1:%.*]] = extractelement <4 x i16> [[VEC4]], i64 1
 ; VI-NEXT:    [[ELT2:%.*]] = extractelement <4 x i16> [[VEC4]], i64 2
 ; VI-NEXT:    [[ELT3:%.*]] = extractelement <4 x i16> [[VEC4]], i64 3
-; VI-NEXT:    [[CMP1:%.*]] = icmp ult i16 [[ELT1]], [[ELT0]]
-; VI-NEXT:    [[MIN1:%.*]] = select i1 [[CMP1]], i16 [[ELT1]], i16 [[ELT0]]
-; VI-NEXT:    [[CMP2:%.*]] = icmp ult i16 [[ELT2]], [[MIN1]]
-; VI-NEXT:    [[MIN2:%.*]] = select i1 [[CMP2]], i16 [[ELT2]], i16 [[MIN1]]
-; VI-NEXT:    [[CMP3:%.*]] = icmp ult i16 [[ELT3]], [[MIN2]]
-; VI-NEXT:    [[MIN3:%.*]] = select i1 [[CMP3]], i16 [[ELT3]], i16 [[MIN2]]
+; VI-NEXT:    [[MIN1:%.*]] = call i16 @llvm.umin.i16(i16 [[ELT1]], i16 [[ELT0]])
+; VI-NEXT:    [[MIN2:%.*]] = call i16 @llvm.umin.i16(i16 [[ELT2]], i16 [[MIN1]])
+; VI-NEXT:    [[MIN3:%.*]] = call i16 @llvm.umin.i16(i16 [[ELT3]], i16 [[MIN2]])
 ; VI-NEXT:    ret i16 [[MIN3]]
 ;
 entry:
@@ -283,48 +280,38 @@ entry:
   %elt1 = extractelement <4 x i16> %vec4, i64 1
   %elt2 = extractelement <4 x i16> %vec4, i64 2
   %elt3 = extractelement <4 x i16> %vec4, i64 3
-
-  %cmp1 = icmp ult i16 %elt1, %elt0
-  %min1 = select i1 %cmp1, i16 %elt1, i16 %elt0
-  %cmp2 = icmp ult i16 %elt2, %min1
-  %min2 = select i1 %cmp2, i16 %elt2, i16 %min1
-  %cmp3 = icmp ult i16 %elt3, %min2
-  %min3 = select i1 %cmp3, i16 %elt3, i16 %min2
-
+  %min1 = call i16 @llvm.umin.i16(i16 %elt1, i16 %elt0)
+  %min2 = call i16 @llvm.umin.i16(i16 %elt2, i16 %min1)
+  %min3 = call i16 @llvm.umin.i16(i16 %elt3, i16 %min2)
   ret i16 %min3
 }
 
 define i16 @reduction_icmp_v8i16(<8 x i16> %vec8) {
-; GFX9-LABEL: @reduction_icmp_v8i16(
-; GFX9-NEXT:  entry:
-; GFX9-NEXT:    [[TMP0:%.*]] = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> [[VEC8:%.*]])
-; GFX9-NEXT:    ret i16 [[TMP0]]
-;
-; VI-LABEL: @reduction_icmp_v8i16(
-; VI-NEXT:  entry:
-; VI-NEXT:    [[ELT0:%.*]] = extractelement <8 x i16> [[VEC8:%.*]], i64 0
-; VI-NEXT:    [[ELT1:%.*]] = extractelement <8 x i16> [[VEC8]], i64 1
-; VI-NEXT:    [[ELT2:%.*]] = extractelement <8 x i16> [[VEC8]], i64 2
-; VI-NEXT:    [[ELT3:%.*]] = extractelement <8 x i16> [[VEC8]], i64 3
-; VI-NEXT:    [[ELT4:%.*]] = extractelement <8 x i16> [[VEC8]], i64 4
-; VI-NEXT:    [[ELT5:%.*]] = extractelement <8 x i16> [[VEC8]], i64 5
-; VI-NEXT:    [[ELT6:%.*]] = extractelement <8 x i16> [[VEC8]], i64 6
-; VI-NEXT:    [[ELT7:%.*]] = extractelement <8 x i16> [[VEC8]], i64 7
-; VI-NEXT:    [[CMP0:%.*]] = icmp ult i16 [[ELT1]], [[ELT0]]
-; VI-NEXT:    [[MIN1:%.*]] = select i1 [[CMP0]], i16 [[ELT1]], i16 [[ELT0]]
-; VI-NEXT:    [[CMP1:%.*]] = icmp ult i16 [[ELT2]], [[MIN1]]
-; VI-NEXT:    [[MIN2:%.*]] = select i1 [[CMP1]], i16 [[ELT2]], i16 [[MIN1]]
-; VI-NEXT:    [[CMP2:%.*]] = icmp ult i16 [[ELT3]], [[MIN2]]
-; VI-NEXT:    [[MIN3:%.*]] = select i1 [[CMP2]], i16 [[ELT3]], i16 [[MIN2]]
-; VI-NEXT:    [[CMP3:%.*]] = icmp ult i16 [[ELT4]], [[MIN3]]
-; VI-NEXT:    [[MIN4:%.*]] = select i1 [[CMP3]], i16 [[ELT4]], i16 [[MIN3]]
-; VI-NEXT:    [[CMP4:%.*]] = icmp ult i16 [[ELT5]], [[MIN4]]
-; VI-NEXT:    [[MIN5:%.*]] = select i1 [[CMP4]], i16 [[ELT5]], i16 [[MIN4]]
-; VI-NEXT:    [[CMP5:%.*]] = icmp ult i16 [[ELT6]], [[MIN5]]
-; VI-NEXT:    [[MIN6:%.*]] = select i1 [[CMP5]], i16 [[ELT6]], i16 [[MIN5]]
-; VI-NEXT:    [[CMP6:%.*]] = icmp ult i16 [[ELT7]], [[MIN6]]
-; VI-NEXT:    [[MIN7:%.*]] = select i1 [[CMP6]], i16 [[ELT7]], i16 [[MIN6]]
-; VI-NEXT:    ret i16 [[MIN7]]
+; GCN-LABEL: @reduction_icmp_v8i16(
+; GCN-NEXT:  entry:
+; GCN-NEXT:    [[ELT0:%.*]] = extractelement <8 x i16> [[VEC8:%.*]], i64 0
+; GCN-NEXT:    [[ELT1:%.*]] = extractelement <8 x i16> [[VEC8]], i64 1
+; GCN-NEXT:    [[ELT2:%.*]] = extractelement <8 x i16> [[VEC8]], i64 2
+; GCN-NEXT:    [[ELT3:%.*]] = extractelement <8 x i16> [[VEC8]], i64 3
+; GCN-NEXT:    [[ELT4:%.*]] = extractelement <8 x i16> [[VEC8]], i64 4
+; GCN-NEXT:    [[ELT5:%.*]] = extractelement <8 x i16> [[VEC8]], i64 5
+; GCN-NEXT:    [[ELT6:%.*]] = extractelement <8 x i16> [[VEC8]], i64 6
+; GCN-NEXT:    [[ELT7:%.*]] = extractelement <8 x i16> [[VEC8]], i64 7
+; GCN-NEXT:    [[CMP0:%.*]] = icmp ult i16 [[ELT1]], [[ELT0]]
+; GCN-NEXT:    [[MIN1:%.*]] = select i1 [[CMP0]], i16 [[ELT1]], i16 [[ELT0]]
+; GCN-NEXT:    [[CMP1:%.*]] = icmp ult i16 [[ELT2]], [[MIN1]]
+; GCN-NEXT:    [[MIN2:%.*]] = select i1 [[CMP1]], i16 [[ELT2]], i16 [[MIN1]]
+; GCN-NEXT:    [[CMP2:%.*]] = icmp ult i16 [[ELT3]], [[MIN2]]
+; GCN-NEXT:    [[MIN3:%.*]] = select i1 [[CMP2]], i16 [[ELT3]], i16 [[MIN2]]
+; GCN-NEXT:    [[CMP3:%.*]] = icmp ult i16 [[ELT4]], [[MIN3]]
+; GCN-NEXT:    [[MIN4:%.*]] = select i1 [[CMP3]], i16 [[ELT4]], i16 [[MIN3]]
+; GCN-NEXT:    [[CMP4:%.*]] = icmp ult i16 [[ELT5]], [[MIN4]]
+; GCN-NEXT:    [[MIN5:%.*]] = select i1 [[CMP4]], i16 [[ELT5]], i16 [[MIN4]]
+; GCN-NEXT:    [[CMP5:%.*]] = icmp ult i16 [[ELT6]], [[MIN5]]
+; GCN-NEXT:    [[MIN6:%.*]] = select i1 [[CMP5]], i16 [[ELT6]], i16 [[MIN5]]
+; GCN-NEXT:    [[CMP6:%.*]] = icmp ult i16 [[ELT7]], [[MIN6]]
+; GCN-NEXT:    [[MIN7:%.*]] = select i1 [[CMP6]], i16 [[ELT7]], i16 [[MIN6]]
+; GCN-NEXT:    ret i16 [[MIN7]]
 ;
 entry:
   %elt0 = extractelement <8 x i16> %vec8, i64 0
@@ -380,36 +367,21 @@ define i16 @reduction_smin_v16i16(<16 x i16> %vec16) {
 ; VI-NEXT:    [[ELT13:%.*]] = extractelement <16 x i16> [[VEC16]], i64 13
 ; VI-NEXT:    [[ELT14:%.*]] = extractelement <16 x i16> [[VEC16]], i64 14
 ; VI-NEXT:    [[ELT15:%.*]] = extractelement <16 x i16> [[VEC16]], i64 15
-; VI-NEXT:    [[CMP0:%.*]] = icmp slt i16 [[ELT1]], [[ELT0]]
-; VI-NEXT:    [[MIN1:%.*]] = select i1 [[CMP0]], i16 [[ELT1]], i16 [[ELT0]]
-; VI-NEXT:    [[CMP1:%.*]] = icmp slt i16 [[ELT2]], [[MIN1]]
-; VI-NEXT:    [[MIN2:%.*]] = select i1 [[CMP1]], i16 [[ELT2]], i16 [[MIN1]]
-; VI-NEXT:    [[CMP2:%.*]] = icmp slt i16 [[ELT3]], [[MIN2]]
-; VI-NEXT:    [[MIN3:%.*]] = select i1 [[CMP2]], i16 [[ELT3]], i16 [[MIN2]]
-; VI-NEXT:    [[CMP3:%.*]] = icmp slt i16 [[ELT4]], [[MIN3]]
-; VI-NEXT:    [[MIN4:%.*]] = select i1 [[CMP3]], i16 [[ELT4]], i16 [[MIN3]]
-; VI-NEXT:    [[CMP4:%.*]] = icmp slt i16 [[ELT5]], [[MIN4]]
-; VI-NEXT:    [[MIN5:%.*]] = select i1 [[CMP4]], i16 [[ELT5]], i16 [[MIN4]]
-; VI-NEXT:    [[CMP5:%.*]] = icmp slt i16 [[ELT6]], [[MIN5]]
-; VI-NEXT:    [[MIN6:%.*]] = select i1 [[CMP5]], i16 [[ELT6]], i16 [[MIN5]]
-; VI-NEXT:    [[CMP6:%.*]] = icmp slt i16 [[ELT7]], [[MIN6]]
-; VI-NEXT:    [[MIN7:%.*]] = select i1 [[CMP6]], i16 [[ELT7]], i16 [[MIN6]]
-; VI-NEXT:    [[CMP7:%.*]] = icmp slt i16 [[ELT8]], [[MIN7]]
-; VI-NEXT:    [[MIN8:%.*]] = select i1 [[CMP7]], i16 [[ELT8]], i16 [[MIN7]]
-; VI-NEXT:    [[CMP8:%.*]] = icmp slt i16 [[ELT9]], [[MIN8]]
-; VI-NEXT:    [[MIN9:%.*]] = select i1 [[CMP8]], i16 [[ELT9]], i16 [[MIN8]]
-; VI-NEXT:    [[CMP9:%.*]] = icmp slt i16 [[ELT10]], [[MIN9]]
-; VI-NEXT:    [[MIN10:%.*]] = select i1 [[CMP9]], i16 [[ELT10]], i16 [[MIN9]]
-; VI-NEXT:    [[CMP10:%.*]] = icmp slt i16 [[ELT11]], [[MIN10]]
-; VI-NEXT:    [[MIN11:%.*]] = select i1 [[CMP10]], i16 [[ELT11]], i16 [[MIN10]]
-; VI-NEXT:    [[CMP11:%.*]] = icmp slt i16 [[ELT12]], [[MIN11]]
-; VI-NEXT:    [[MIN12:%.*]] = select i1 [[CMP11]], i16 [[ELT12]], i16 [[MIN11]]
-; VI-NEXT:    [[CMP12:%.*]] = icmp slt i16 [[ELT13]], [[MIN12]]
-; VI-NEXT:    [[MIN13:%.*]] = select i1 [[CMP12]], i16 [[ELT13]], i16 [[MIN12]]
-; VI-NEXT:    [[CMP13:%.*]] = icmp slt i16 [[ELT14]], [[MIN13]]
-; VI-NEXT:    [[MIN14:%.*]] = select i1 [[CMP13]], i16 [[ELT14]], i16 [[MIN13]]
-; VI-NEXT:    [[CMP14:%.*]] = icmp slt i16 [[ELT15]], [[MIN14]]
-; VI-NEXT:    [[MIN15:%.*]] = select i1 [[CMP14]], i16 [[ELT15]], i16 [[MIN14]]
+; VI-NEXT:    [[MIN1:%.*]] = call i16 @llvm.smin.i16(i16 [[ELT1]], i16 [[ELT0]])
+; VI-NEXT:    [[MIN2:%.*]] = call i16 @llvm.smin.i16(i16 [[ELT2]], i16 [[MIN1]])
+; VI-NEXT:    [[MIN3:%.*]] = call i16 @llvm.smin.i16(i16 [[ELT3]], i16 [[MIN2]])
+; VI-NEXT:    [[MIN4:%.*]] = call i16 @llvm.smin.i16(i16 [[ELT4]], i16 [[MIN3]])
+; VI-NEXT:    [[MIN5:%.*]] = call i16 @llvm.smin.i16(i16 [[ELT5]], i16 [[MIN4]])
+; VI-NEXT:    [[MIN6:%.*]] = call i16 @llvm.smin.i16(i16 [[ELT6]], i16 [[MIN5]])
+; VI-NEXT:    [[MIN7:%.*]] = call i16 @llvm.smin.i16(i16 [[ELT7]], i16 [[MIN6]])
+; VI-NEXT:    [[MIN8:%.*]] = call i16 @llvm.smin.i16(i16 [[ELT8]], i16 [[MIN7]])
+; VI-NEXT:    [[MIN9:%.*]] = call i16 @llvm.smin.i16(i16 [[ELT9]], i16 [[MIN8]])
+; VI-NEXT:    [[MIN10:%.*]] = call i16 @llvm.smin.i16(i16 [[ELT10]], i16 [[MIN9]])
+; VI-NEXT:    [[MIN11:%.*]] = call i16 @llvm.smin.i16(i16 [[ELT11]], i16 [[MIN10]])
+; VI-NEXT:    [[MIN12:%.*]] = call i16 @llvm.smin.i16(i16 [[ELT12]], i16 [[MIN11]])
+; VI-NEXT:    [[MIN13:%.*]] = call i16 @llvm.smin.i16(i16 [[ELT13]], i16 [[MIN12]])
+; VI-NEXT:    [[MIN14:%.*]] = call i16 @llvm.smin.i16(i16 [[ELT14]], i16 [[MIN13]])
+; VI-NEXT:    [[MIN15:%.*]] = call i16 @llvm.smin.i16(i16 [[ELT15]], i16 [[MIN14]])
 ; VI-NEXT:    ret i16 [[MIN15]]
 ;
 entry:
@@ -430,45 +402,21 @@ entry:
   %elt13 = extractelement <16 x i16> %vec16, i64 13
   %elt14 = extractelement <16 x i16> %vec16, i64 14
   %elt15 = extractelement <16 x i16> %vec16, i64 15
-
-  %cmp0 = icmp slt i16 %elt1, %elt0
-  %min1 = select i1 %cmp0, i16 %elt1, i16 %elt0
-  %cmp1 = icmp slt i16 %elt2, %min1
-  %min2 = select i1 %cmp1, i16 %elt2, i16 %min1
-  %cmp2 = icmp slt i16 %elt3, %min2
-  %min3 = select i1 %cmp2, i16 %elt3, i16 %min2
-
-  %cmp3 = icmp slt i16 %elt4, %min3
-  %min4 = select i1 %cmp3, i16 %elt4, i16 %min3
-  %cmp4 = icmp slt i16 %elt5, %min4
-  %min5 = select i1 %cmp4, i16 %elt5, i16 %min4
-
-  %cmp5 = icmp slt i16 %elt6, %min5
-  %min6 = select i1 %cmp5, i16 %elt6, i16 %min5
-  %cmp6 = icmp slt i16 %elt7, %min6
-  %min7 = select i1 %cmp6, i16 %elt7, i16 %min6
-
-  %cmp7 = icmp slt i16 %elt8, %min7
-  %min8 = select i1 %cmp7, i16 %elt8, i16 %min7
-  %cmp8 = icmp slt i16 %elt9, %min8
-  %min9 = select i1 %cmp8, i16 %elt9, i16 %min8
-
-  %cmp9 = icmp slt i16 %elt10, %min9
-  %min10 = select i1 %cmp9, i16 %elt10, i16 %min9
-  %cmp10 = icmp slt i16 %elt11, %min10
-  %min11 = select i1 %cmp10, i16 %elt11, i16 %min10
-
-  %cmp11 = icmp slt i16 %elt12, %min11
-  %min12 = select i1 %cmp11, i16 %elt12, i16 %min11
-  %cmp12 = icmp slt i16 %elt13, %min12
-  %min13 = select i1 %cmp12, i16 %elt13, i16 %min12
-
-  %cmp13 = icmp slt i16 %elt14, %min13
-  %min14 = select i1 %cmp13, i16 %elt14, i16 %min13
-  %cmp14 = icmp slt i16 %elt15, %min14
-  %min15 = select i1 %cmp14, i16 %elt15, i16 %min14
-
-
+  %min1 = call i16 @llvm.smin.i16(i16 %elt1, i16 %elt0)
+  %min2 = call i16 @llvm.smin.i16(i16 %elt2, i16 %min1)
+  %min3 = call i16 @llvm.smin.i16(i16 %elt3, i16 %min2)
+  %min4 = call i16 @llvm.smin.i16(i16 %elt4, i16 %min3)
+  %min5 = call i16 @llvm.smin.i16(i16 %elt5, i16 %min4)
+  %min6 = call i16 @llvm.smin.i16(i16 %elt6, i16 %min5)
+  %min7 = call i16 @llvm.smin.i16(i16 %elt7, i16 %min6)
+  %min8 = call i16 @llvm.smin.i16(i16 %elt8, i16 %min7)
+  %min9 = call i16 @llvm.smin.i16(i16 %elt9, i16 %min8)
+  %min10 = call i16 @llvm.smin.i16(i16 %elt10, i16 %min9)
+  %min11 = call i16 @llvm.smin.i16(i16 %elt11, i16 %min10)
+  %min12 = call i16 @llvm.smin.i16(i16 %elt12, i16 %min11)
+  %min13 = call i16 @llvm.smin.i16(i16 %elt13, i16 %min12)
+  %min14 = call i16 @llvm.smin.i16(i16 %elt14, i16 %min13)
+  %min15 = call i16 @llvm.smin.i16(i16 %elt15, i16 %min14)
   ret i16 %min15
 }
 
@@ -484,12 +432,9 @@ define i16 @reduction_umax_v4i16(<4 x i16> %vec4) {
 ; VI-NEXT:    [[ELT1:%.*]] = extractelement <4 x i16> [[VEC4]], i64 1
 ; VI-NEXT:    [[ELT2:%.*]] = extractelement <4 x i16> [[VEC4]], i64 2
 ; VI-NEXT:    [[ELT3:%.*]] = extractelement <4 x i16> [[VEC4]], i64 3
-; VI-NEXT:    [[CMP1:%.*]] = icmp ugt i16 [[ELT1]], [[ELT0]]
-; VI-NEXT:    [[MAX1:%.*]] = select i1 [[CMP1]], i16 [[ELT1]], i16 [[ELT0]]
-; VI-NEXT:    [[CMP2:%.*]] = icmp ugt i16 [[ELT2]], [[MAX1]]
-; VI-NEXT:    [[MAX2:%.*]] = select i1 [[CMP2]], i16 [[ELT2]], i16 [[MAX1]]
-; VI-NEXT:    [[CMP3:%.*]] = icmp ugt i16 [[ELT3]], [[MAX2]]
-; VI-NEXT:    [[MAX3:%.*]] = select i1 [[CMP3]], i16 [[ELT3]], i16 [[MAX2]]
+; VI-NEXT:    [[MAX1:%.*]] = call i16 @llvm.umax.i16(i16 [[ELT1]], i16 [[ELT0]])
+; VI-NEXT:    [[MAX2:%.*]] = call i16 @llvm.umax.i16(i16 [[ELT2]], i16 [[MAX1]])
+; VI-NEXT:    [[MAX3:%.*]] = call i16 @llvm.umax.i16(i16 [[ELT3]], i16 [[MAX2]])
 ; VI-NEXT:    ret i16 [[MAX3]]
 ;
 entry:
@@ -497,14 +442,9 @@ entry:
   %elt1 = extractelement <4 x i16> %vec4, i64 1
   %elt2 = extractelement <4 x i16> %vec4, i64 2
   %elt3 = extractelement <4 x i16> %vec4, i64 3
-
-  %cmp1 = icmp ugt i16 %elt1, %elt0
-  %max1 = select i1 %cmp1, i16 %elt1, i16 %elt0
-  %cmp2 = icmp ugt i16 %elt2, %max1
-  %max2 = select i1 %cmp2, i16 %elt2, i16 %max1
-  %cmp3 = icmp ugt i16 %elt3, %max2
-  %max3 = select i1 %cmp3, i16 %elt3, i16 %max2
-
+  %max1 = call i16 @llvm.umax.i16(i16 %elt1, i16 %elt0)
+  %max2 = call i16 @llvm.umax.i16(i16 %elt2, i16 %max1)
+  %max3 = call i16 @llvm.umax.i16(i16 %elt3, i16 %max2)
   ret i16 %max3
 }
 
@@ -520,12 +460,9 @@ define i16 @reduction_smax_v4i16(<4 x i16> %vec4) {
 ; VI-NEXT:    [[ELT1:%.*]] = extractelement <4 x i16> [[VEC4]], i64 1
 ; VI-NEXT:    [[ELT2:%.*]] = extractelement <4 x i16> [[VEC4]], i64 2
 ; VI-NEXT:    [[ELT3:%.*]] = extractelement <4 x i16> [[VEC4]], i64 3
-; VI-NEXT:    [[CMP1:%.*]] = icmp sgt i16 [[ELT1]], [[ELT0]]
-; VI-NEXT:    [[MAX1:%.*]] = select i1 [[CMP1]], i16 [[ELT1]], i16 [[ELT0]]
-; VI-NEXT:    [[CMP2:%.*]] = icmp sgt i16 [[ELT2]], [[MAX1]]
-; VI-NEXT:    [[MAX2:%.*]] = select i1 [[CMP2]], i16 [[ELT2]], i16 [[MAX1]]
-; VI-NEXT:    [[CMP3:%.*]] = icmp sgt i16 [[ELT3]], [[MAX2]]
-; VI-NEXT:    [[MAX3:%.*]] = select i1 [[CMP3]], i16 [[ELT3]], i16 [[MAX2]]
+; VI-NEXT:    [[MAX1:%.*]] = call i16 @llvm.smax.i16(i16 [[ELT1]], i16 [[ELT0]])
+; VI-NEXT:    [[MAX2:%.*]] = call i16 @llvm.smax.i16(i16 [[ELT2]], i16 [[MAX1]])
+; VI-NEXT:    [[MAX3:%.*]] = call i16 @llvm.smax.i16(i16 [[ELT3]], i16 [[MAX2]])
 ; VI-NEXT:    ret i16 [[MAX3]]
 ;
 entry:
@@ -533,14 +470,9 @@ entry:
   %elt1 = extractelement <4 x i16> %vec4, i64 1
   %elt2 = extractelement <4 x i16> %vec4, i64 2
   %elt3 = extractelement <4 x i16> %vec4, i64 3
-
-  %cmp1 = icmp sgt i16 %elt1, %elt0
-  %max1 = select i1 %cmp1, i16 %elt1, i16 %elt0
-  %cmp2 = icmp sgt i16 %elt2, %max1
-  %max2 = select i1 %cmp2, i16 %elt2, i16 %max1
-  %cmp3 = icmp sgt i16 %elt3, %max2
-  %max3 = select i1 %cmp3, i16 %elt3, i16 %max2
-
+  %max1 = call i16 @llvm.smax.i16(i16 %elt1, i16 %elt0)
+  %max2 = call i16 @llvm.smax.i16(i16 %elt2, i16 %max1)
+  %max3 = call i16 @llvm.smax.i16(i16 %elt3, i16 %max2)
   ret i16 %max3
 }
 
diff --git a/llvm/test/Transforms/SLPVectorizer/WebAssembly/or-reduction-zero-test.ll b/llvm/test/Transforms/SLPVectorizer/WebAssembly/or-reduction-zero-test.ll
new file mode 100644
index 0000000000000..b7d0c2694af7f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/WebAssembly/or-reduction-zero-test.ll
@@ -0,0 +1,235 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=wasm32 -mattr=+simd128 -passes=slp-vectorizer -S < %s | FileCheck %s
+
+define i1 @or_reduction_nonzero(ptr %p) {
+; CHECK-LABEL: define i1 @or_reduction_nonzero(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[P]], align 1
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 0, i32 8>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 1, i32 9>
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i8> [[TMP18]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 2, i32 10>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 3, i32 11>
+; CHECK-NEXT:    [[TMP6:%.*]] = or <2 x i8> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 4, i32 12>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 5, i32 13>
+; CHECK-NEXT:    [[TMP9:%.*]] = or <2 x i8> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 6, i32 14>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 7, i32 15>
+; CHECK-NEXT:    [[TMP12:%.*]] = or <2 x i8> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <2 x i8> [[TMP3]], [[TMP6]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <2 x i8> [[TMP9]], [[TMP12]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <2 x i8> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i8> [[TMP15]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i8> [[TMP15]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = or i8 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %v0 = load i8, ptr %p, align 1
+  %p1 = getelementptr inbounds i8, ptr %p, i32 1
+  %v1 = load i8, ptr %p1, align 1
+  %p2 = getelementptr inbounds i8, ptr %p, i32 2
+  %v2 = load i8, ptr %p2, align 1
+  %p3 = getelementptr inbounds i8, ptr %p, i32 3
+  %v3 = load i8, ptr %p3, align 1
+  %p4 = getelementptr inbounds i8, ptr %p, i32 4
+  %v4 = load i8, ptr %p4, align 1
+  %p5 = getelementptr inbounds i8, ptr %p, i32 5
+  %v5 = load i8, ptr %p5, align 1
+  %p6 = getelementptr inbounds i8, ptr %p, i32 6
+  %v6 = load i8, ptr %p6, align 1
+  %p7 = getelementptr inbounds i8, ptr %p, i32 7
+  %v7 = load i8, ptr %p7, align 1
+  %p8 = getelementptr inbounds i8, ptr %p, i32 8
+  %v8 = load i8, ptr %p8, align 1
+  %p9 = getelementptr inbounds i8, ptr %p, i32 9
+  %v9 = load i8, ptr %p9, align 1
+  %p10 = getelementptr inbounds i8, ptr %p, i32 10
+  %v10 = load i8, ptr %p10, align 1
+  %p11 = getelementptr inbounds i8, ptr %p, i32 11
+  %v11 = load i8, ptr %p11, align 1
+  %p12 = getelementptr inbounds i8, ptr %p, i32 12
+  %v12 = load i8, ptr %p12, align 1
+  %p13 = getelementptr inbounds i8, ptr %p, i32 13
+  %v13 = load i8, ptr %p13, align 1
+  %p14 = getelementptr inbounds i8, ptr %p, i32 14
+  %v14 = load i8, ptr %p14, align 1
+  %p15 = getelementptr inbounds i8, ptr %p, i32 15
+  %v15 = load i8, ptr %p15, align 1
+  %o01 = or i8 %v0, %v1
+  %o23 = or i8 %v2, %v3
+  %o45 = or i8 %v4, %v5
+  %o67 = or i8 %v6, %v7
+  %o89 = or i8 %v8, %v9
+  %o1011 = or i8 %v10, %v11
+  %o1213 = or i8 %v12, %v13
+  %o1415 = or i8 %v14, %v15
+  %o0123 = or i8 %o01, %o23
+  %o4567 = or i8 %o45, %o67
+  %o891011 = or i8 %o89, %o1011
+  %o12131415 = or i8 %o1213, %o1415
+  %o07 = or i8 %o0123, %o4567
+  %o815 = or i8 %o891011, %o12131415
+  %o = or i8 %o07, %o815
+  %cmp = icmp ne i8 %o, 0
+  ret i1 %cmp
+}
+
+define i1 @or_reduction_zero(ptr %p) {
+; CHECK-LABEL: define i1 @or_reduction_zero(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[P]], align 1
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 0, i32 8>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 1, i32 9>
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i8> [[TMP18]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 2, i32 10>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 3, i32 11>
+; CHECK-NEXT:    [[TMP6:%.*]] = or <2 x i8> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 4, i32 12>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 5, i32 13>
+; CHECK-NEXT:    [[TMP9:%.*]] = or <2 x i8> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 6, i32 14>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 7, i32 15>
+; CHECK-NEXT:    [[TMP12:%.*]] = or <2 x i8> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <2 x i8> [[TMP3]], [[TMP6]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <2 x i8> [[TMP9]], [[TMP12]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <2 x i8> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i8> [[TMP15]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i8> [[TMP15]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = or i8 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %v0 = load i8, ptr %p, align 1
+  %p1 = getelementptr inbounds i8, ptr %p, i32 1
+  %v1 = load i8, ptr %p1, align 1
+  %p2 = getelementptr inbounds i8, ptr %p, i32 2
+  %v2 = load i8, ptr %p2, align 1
+  %p3 = getelementptr inbounds i8, ptr %p, i32 3
+  %v3 = load i8, ptr %p3, align 1
+  %p4 = getelementptr inbounds i8, ptr %p, i32 4
+  %v4 = load i8, ptr %p4, align 1
+  %p5 = getelementptr inbounds i8, ptr %p, i32 5
+  %v5 = load i8, ptr %p5, align 1
+  %p6 = getelementptr inbounds i8, ptr %p, i32 6
+  %v6 = load i8, ptr %p6, align 1
+  %p7 = getelementptr inbounds i8, ptr %p, i32 7
+  %v7 = load i8, ptr %p7, align 1
+  %p8 = getelementptr inbounds i8, ptr %p, i32 8
+  %v8 = load i8, ptr %p8, align 1
+  %p9 = getelementptr inbounds i8, ptr %p, i32 9
+  %v9 = load i8, ptr %p9, align 1
+  %p10 = getelementptr inbounds i8, ptr %p, i32 10
+  %v10 = load i8, ptr %p10, align 1
+  %p11 = getelementptr inbounds i8, ptr %p, i32 11
+  %v11 = load i8, ptr %p11, align 1
+  %p12 = getelementptr inbounds i8, ptr %p, i32 12
+  %v12 = load i8, ptr %p12, align 1
+  %p13 = getelementptr inbounds i8, ptr %p, i32 13
+  %v13 = load i8, ptr %p13, align 1
+  %p14 = getelementptr inbounds i8, ptr %p, i32 14
+  %v14 = load i8, ptr %p14, align 1
+  %p15 = getelementptr inbounds i8, ptr %p, i32 15
+  %v15 = load i8, ptr %p15, align 1
+  %o01 = or i8 %v0, %v1
+  %o23 = or i8 %v2, %v3
+  %o45 = or i8 %v4, %v5
+  %o67 = or i8 %v6, %v7
+  %o89 = or i8 %v8, %v9
+  %o1011 = or i8 %v10, %v11
+  %o1213 = or i8 %v12, %v13
+  %o1415 = or i8 %v14, %v15
+  %o0123 = or i8 %o01, %o23
+  %o4567 = or i8 %o45, %o67
+  %o891011 = or i8 %o89, %o1011
+  %o12131415 = or i8 %o1213, %o1415
+  %o07 = or i8 %o0123, %o4567
+  %o815 = or i8 %o891011, %o12131415
+  %o = or i8 %o07, %o815
+  %cmp = icmp eq i8 %o, 0
+  ret i1 %cmp
+}
+
+define i1 @or_reduction_nonzero_multiuse(ptr %p, ptr %out) {
+; CHECK-LABEL: define i1 @or_reduction_nonzero_multiuse(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[P]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 0, i32 8>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 1, i32 9>
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 2, i32 10>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 3, i32 11>
+; CHECK-NEXT:    [[TMP6:%.*]] = or <2 x i8> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 4, i32 12>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 5, i32 13>
+; CHECK-NEXT:    [[TMP9:%.*]] = or <2 x i8> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 6, i32 14>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 7, i32 15>
+; CHECK-NEXT:    [[TMP12:%.*]] = or <2 x i8> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <2 x i8> [[TMP3]], [[TMP6]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <2 x i8> [[TMP9]], [[TMP12]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <2 x i8> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i8> [[TMP15]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i8> [[TMP15]], i32 1
+; CHECK-NEXT:    [[O:%.*]] = or i8 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    store i8 [[O]], ptr [[OUT]], align 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[O]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %v0 = load i8, ptr %p, align 1
+  %p1 = getelementptr inbounds i8, ptr %p, i32 1
+  %v1 = load i8, ptr %p1, align 1
+  %p2 = getelementptr inbounds i8, ptr %p, i32 2
+  %v2 = load i8, ptr %p2, align 1
+  %p3 = getelementptr inbounds i8, ptr %p, i32 3
+  %v3 = load i8, ptr %p3, align 1
+  %p4 = getelementptr inbounds i8, ptr %p, i32 4
+  %v4 = load i8, ptr %p4, align 1
+  %p5 = getelementptr inbounds i8, ptr %p, i32 5
+  %v5 = load i8, ptr %p5, align 1
+  %p6 = getelementptr inbounds i8, ptr %p, i32 6
+  %v6 = load i8, ptr %p6, align 1
+  %p7 = getelementptr inbounds i8, ptr %p, i32 7
+  %v7 = load i8, ptr %p7, align 1
+  %p8 = getelementptr inbounds i8, ptr %p, i32 8
+  %v8 = load i8, ptr %p8, align 1
+  %p9 = getelementptr inbounds i8, ptr %p, i32 9
+  %v9 = load i8, ptr %p9, align 1
+  %p10 = getelementptr inbounds i8, ptr %p, i32 10
+  %v10 = load i8, ptr %p10, align 1
+  %p11 = getelementptr inbounds i8, ptr %p, i32 11
+  %v11 = load i8, ptr %p11, align 1
+  %p12 = getelementptr inbounds i8, ptr %p, i32 12
+  %v12 = load i8, ptr %p12, align 1
+  %p13 = getelementptr inbounds i8, ptr %p, i32 13
+  %v13 = load i8, ptr %p13, align 1
+  %p14 = getelementptr inbounds i8, ptr %p, i32 14
+  %v14 = load i8, ptr %p14, align 1
+  %p15 = getelementptr inbounds i8, ptr %p, i32 15
+  %v15 = load i8, ptr %p15, align 1
+  %o01 = or i8 %v0, %v1
+  %o23 = or i8 %v2, %v3
+  %o45 = or i8 %v4, %v5
+  %o67 = or i8 %v6, %v7
+  %o89 = or i8 %v8, %v9
+  %o1011 = or i8 %v10, %v11
+  %o1213 = or i8 %v12, %v13
+  %o1415 = or i8 %v14, %v15
+  %o0123 = or i8 %o01, %o23
+  %o4567 = or i8 %o45, %o67
+  %o891011 = or i8 %o89, %o1011
+  %o12131415 = or i8 %o1213, %o1415
+  %o07 = or i8 %o0123, %o4567
+  %o815 = or i8 %o891011, %o12131415
+  %o = or i8 %o07, %o815
+  store i8 %o, ptr %out, align 1
+  %cmp = icmp ne i8 %o, 0
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extractelement-single-use-many-nodes.ll b/llvm/test/Transforms/SLPVectorizer/X86/extractelement-single-use-many-nodes.ll
index 91ec61b275205..31e4feabd3582 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extractelement-single-use-many-nodes.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extractelement-single-use-many-nodes.ll
@@ -33,8 +33,7 @@ define void @foo(double %i) {
 ; CHECK-NEXT:    [[TMP29:%.*]] = fptosi <4 x double> [[TMP28]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP30:%.*]] = or <4 x i32> zeroinitializer, [[TMP29]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP30]])
-; CHECK-NEXT:    [[OP_RDX:%.*]] = icmp slt i32 [[TMP31]], 32000
-; CHECK-NEXT:    [[OP_RDX1:%.*]] = select i1 [[OP_RDX]], i32 [[TMP31]], i32 32000
+; CHECK-NEXT:    [[OP_RDX1:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP31]], i32 32000)
 ; CHECK-NEXT:    [[I163:%.*]] = fcmp ogt double [[I118]], 0.000000e+00
 ; CHECK-NEXT:    [[I164:%.*]] = icmp slt i32 0, [[OP_RDX1]]
 ; CHECK-NEXT:    ret void
@@ -105,29 +104,25 @@ bb115:
   %i134 = fmul double %i133, 0.000000e+00
   %i135 = fptosi double %i134 to i32
   %i136 = or i32 0, %i135
-  %i137 = icmp slt i32 %i136, 32000
-  %i138 = select i1 %i137, i32 %i136, i32 32000
+  %i138 = call i32 @llvm.smin(i32 %i136, i32 32000)
   %i139 = select i1 false, double 0.000000e+00, double %i130
   %i140 = fmul double %i139, 0.000000e+00
   %i141 = fmul double %i140, 0.000000e+00
   %i142 = fptosi double %i141 to i32
   %i143 = or i32 0, %i142
-  %i144 = icmp slt i32 %i143, %i138
-  %i145 = select i1 %i144, i32 %i143, i32 %i138
+  %i145 = call i32 @llvm.smin(i32 %i143, i32 %i138)
   %i146 = select i1 false, double 0.000000e+00, double %i129
   %i147 = fmul double %i146, 0.000000e+00
   %i148 = fmul double %i147, 0.000000e+00
   %i149 = fptosi double %i148 to i32
   %i150 = or i32 0, %i149
-  %i151 = icmp slt i32 %i150, %i145
-  %i152 = select i1 %i151, i32 %i150, i32 %i145
+  %i152 = call i32 @llvm.smin(i32 %i150, i32 %i145)
   %i153 = select i1 false, double 0.000000e+00, double %i126
   %i154 = fmul double %i153, 0.000000e+00
   %i155 = fmul double %i154, 0.000000e+00
   %i156 = fptosi double %i155 to i32
   %i157 = or i32 0, %i156
-  %i158 = icmp slt i32 %i157, %i152
-  %i159 = select i1 %i158, i32 %i157, i32 %i152
+  %i159 = call i32 @llvm.smin(i32 %i157, i32 %i152)
   %i163 = fcmp ogt double %i118, 0.000000e+00
   %i164 = icmp slt i32 0, %i159
   ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
index b2b9363565277..a059c9bdbcbcd 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
@@ -24,28 +24,21 @@ define i32 @maxi8(i32) {
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
   %2 = load i32, ptr @arr, align 16
-  %3 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 1), align 4
-  %4 = icmp sgt i32 %2, %3
-  %5 = select i1 %4, i32 %2, i32 %3
-  %6 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 2), align 8
-  %7 = icmp sgt i32 %5, %6
-  %8 = select i1 %7, i32 %5, i32 %6
-  %9 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 3), align 4
-  %10 = icmp sgt i32 %8, %9
-  %11 = select i1 %10, i32 %8, i32 %9
-  %12 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 4), align 16
-  %13 = icmp sgt i32 %11, %12
-  %14 = select i1 %13, i32 %11, i32 %12
-  %15 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 5), align 4
-  %16 = icmp sgt i32 %14, %15
-  %17 = select i1 %16, i32 %14, i32 %15
-  %18 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 6), align 8
-  %19 = icmp sgt i32 %17, %18
-  %20 = select i1 %19, i32 %17, i32 %18
-  %21 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 7), align 4
-  %22 = icmp sgt i32 %20, %21
-  %23 = select i1 %22, i32 %20, i32 %21
-  ret i32 %23
+  %3 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 4), align 4
+  %4 = call i32 @llvm.smax.i32(i32 %2, i32 %3)
+  %5 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 8), align 8
+  %6 = call i32 @llvm.smax.i32(i32 %4, i32 %5)
+  %7 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 12), align 4
+  %8 = call i32 @llvm.smax.i32(i32 %6, i32 %7)
+  %9 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 16), align 16
+  %10 = call i32 @llvm.smax.i32(i32 %8, i32 %9)
+  %11 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 20), align 4
+  %12 = call i32 @llvm.smax.i32(i32 %10, i32 %11)
+  %13 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 24), align 8
+  %14 = call i32 @llvm.smax.i32(i32 %12, i32 %13)
+  %15 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 28), align 4
+  %16 = call i32 @llvm.smax.i32(i32 %14, i32 %15)
+  ret i32 %16
 }
 
 define i32 @maxi8_store_in(i32) {
@@ -56,29 +49,22 @@ define i32 @maxi8_store_in(i32) {
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
   %2 = load i32, ptr @arr, align 16
-  %3 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 1), align 4
-  %4 = icmp sgt i32 %2, %3
-  %5 = select i1 %4, i32 %2, i32 %3
-  %6 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 2), align 8
-  %7 = icmp sgt i32 %5, %6
-  %8 = select i1 %7, i32 %5, i32 %6
-  %9 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 3), align 4
-  %10 = icmp sgt i32 %8, %9
-  %11 = select i1 %10, i32 %8, i32 %9
-  %12 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 4), align 16
-  %13 = icmp sgt i32 %11, %12
-  %14 = select i1 %13, i32 %11, i32 %12
+  %3 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 4), align 4
+  %4 = call i32 @llvm.smax.i32(i32 %2, i32 %3)
+  %5 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 8), align 8
+  %6 = call i32 @llvm.smax.i32(i32 %4, i32 %5)
+  %7 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 12), align 4
+  %8 = call i32 @llvm.smax.i32(i32 %6, i32 %7)
+  %9 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 16), align 16
+  %10 = call i32 @llvm.smax.i32(i32 %8, i32 %9)
   store i32 0, ptr @var, align 8
-  %15 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 5), align 4
-  %16 = icmp sgt i32 %14, %15
-  %17 = select i1 %16, i32 %14, i32 %15
-  %18 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 6), align 8
-  %19 = icmp sgt i32 %17, %18
-  %20 = select i1 %19, i32 %17, i32 %18
-  %21 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 7), align 4
-  %22 = icmp sgt i32 %20, %21
-  %23 = select i1 %22, i32 %20, i32 %21
-  ret i32 %23
+  %11 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 20), align 4
+  %12 = call i32 @llvm.smax.i32(i32 %10, i32 %11)
+  %13 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 24), align 8
+  %14 = call i32 @llvm.smax.i32(i32 %12, i32 %13)
+  %15 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 28), align 4
+  %16 = call i32 @llvm.smax.i32(i32 %14, i32 %15)
+  ret i32 %16
 }
 
 define i32 @maxi16(i32) {
@@ -88,52 +74,37 @@ define i32 @maxi16(i32) {
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
   %2 = load i32, ptr @arr, align 16
-  %3 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 1), align 4
-  %4 = icmp sgt i32 %2, %3
-  %5 = select i1 %4, i32 %2, i32 %3
-  %6 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 2), align 8
-  %7 = icmp sgt i32 %5, %6
-  %8 = select i1 %7, i32 %5, i32 %6
-  %9 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 3), align 4
-  %10 = icmp sgt i32 %8, %9
-  %11 = select i1 %10, i32 %8, i32 %9
-  %12 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 4), align 16
-  %13 = icmp sgt i32 %11, %12
-  %14 = select i1 %13, i32 %11, i32 %12
-  %15 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 5), align 4
-  %16 = icmp sgt i32 %14, %15
-  %17 = select i1 %16, i32 %14, i32 %15
-  %18 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 6), align 8
-  %19 = icmp sgt i32 %17, %18
-  %20 = select i1 %19, i32 %17, i32 %18
-  %21 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 7), align 4
-  %22 = icmp sgt i32 %20, %21
-  %23 = select i1 %22, i32 %20, i32 %21
-  %24 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 8), align 16
-  %25 = icmp sgt i32 %23, %24
-  %26 = select i1 %25, i32 %23, i32 %24
-  %27 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 9), align 4
-  %28 = icmp sgt i32 %26, %27
-  %29 = select i1 %28, i32 %26, i32 %27
-  %30 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 10), align 8
-  %31 = icmp sgt i32 %29, %30
-  %32 = select i1 %31, i32 %29, i32 %30
-  %33 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 11), align 4
-  %34 = icmp sgt i32 %32, %33
-  %35 = select i1 %34, i32 %32, i32 %33
-  %36 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 12), align 16
-  %37 = icmp sgt i32 %35, %36
-  %38 = select i1 %37, i32 %35, i32 %36
-  %39 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 13), align 4
-  %40 = icmp sgt i32 %38, %39
-  %41 = select i1 %40, i32 %38, i32 %39
-  %42 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 14), align 8
-  %43 = icmp sgt i32 %41, %42
-  %44 = select i1 %43, i32 %41, i32 %42
-  %45 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 15), align 4
-  %46 = icmp sgt i32 %44, %45
-  %47 = select i1 %46, i32 %44, i32 %45
-  ret i32 %47
+  %3 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 4), align 4
+  %4 = call i32 @llvm.smax.i32(i32 %2, i32 %3)
+  %5 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 8), align 8
+  %6 = call i32 @llvm.smax.i32(i32 %4, i32 %5)
+  %7 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 12), align 4
+  %8 = call i32 @llvm.smax.i32(i32 %6, i32 %7)
+  %9 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 16), align 16
+  %10 = call i32 @llvm.smax.i32(i32 %8, i32 %9)
+  %11 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 20), align 4
+  %12 = call i32 @llvm.smax.i32(i32 %10, i32 %11)
+  %13 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 24), align 8
+  %14 = call i32 @llvm.smax.i32(i32 %12, i32 %13)
+  %15 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 28), align 4
+  %16 = call i32 @llvm.smax.i32(i32 %14, i32 %15)
+  %17 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 32), align 16
+  %18 = call i32 @llvm.smax.i32(i32 %16, i32 %17)
+  %19 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 36), align 4
+  %20 = call i32 @llvm.smax.i32(i32 %18, i32 %19)
+  %21 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 40), align 8
+  %22 = call i32 @llvm.smax.i32(i32 %20, i32 %21)
+  %23 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 44), align 4
+  %24 = call i32 @llvm.smax.i32(i32 %22, i32 %23)
+  %25 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 48), align 16
+  %26 = call i32 @llvm.smax.i32(i32 %24, i32 %25)
+  %27 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 52), align 4
+  %28 = call i32 @llvm.smax.i32(i32 %26, i32 %27)
+  %29 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 56), align 8
+  %30 = call i32 @llvm.smax.i32(i32 %28, i32 %29)
+  %31 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 60), align 4
+  %32 = call i32 @llvm.smax.i32(i32 %30, i32 %31)
+  ret i32 %32
 }
 
 define i32 @maxi32(i32) {
@@ -143,100 +114,69 @@ define i32 @maxi32(i32) {
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
   %2 = load i32, ptr @arr, align 16
-  %3 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 1), align 4
-  %4 = icmp sgt i32 %2, %3
-  %5 = select i1 %4, i32 %2, i32 %3
-  %6 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 2), align 8
-  %7 = icmp sgt i32 %5, %6
-  %8 = select i1 %7, i32 %5, i32 %6
-  %9 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 3), align 4
-  %10 = icmp sgt i32 %8, %9
-  %11 = select i1 %10, i32 %8, i32 %9
-  %12 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 4), align 16
-  %13 = icmp sgt i32 %11, %12
-  %14 = select i1 %13, i32 %11, i32 %12
-  %15 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 5), align 4
-  %16 = icmp sgt i32 %14, %15
-  %17 = select i1 %16, i32 %14, i32 %15
-  %18 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 6), align 8
-  %19 = icmp sgt i32 %17, %18
-  %20 = select i1 %19, i32 %17, i32 %18
-  %21 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 7), align 4
-  %22 = icmp sgt i32 %20, %21
-  %23 = select i1 %22, i32 %20, i32 %21
-  %24 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 8), align 16
-  %25 = icmp sgt i32 %23, %24
-  %26 = select i1 %25, i32 %23, i32 %24
-  %27 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 9), align 4
-  %28 = icmp sgt i32 %26, %27
-  %29 = select i1 %28, i32 %26, i32 %27
-  %30 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 10), align 8
-  %31 = icmp sgt i32 %29, %30
-  %32 = select i1 %31, i32 %29, i32 %30
-  %33 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 11), align 4
-  %34 = icmp sgt i32 %32, %33
-  %35 = select i1 %34, i32 %32, i32 %33
-  %36 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 12), align 16
-  %37 = icmp sgt i32 %35, %36
-  %38 = select i1 %37, i32 %35, i32 %36
-  %39 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 13), align 4
-  %40 = icmp sgt i32 %38, %39
-  %41 = select i1 %40, i32 %38, i32 %39
-  %42 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 14), align 8
-  %43 = icmp sgt i32 %41, %42
-  %44 = select i1 %43, i32 %41, i32 %42
-  %45 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 15), align 4
-  %46 = icmp sgt i32 %44, %45
-  %47 = select i1 %46, i32 %44, i32 %45
-  %48 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 16), align 16
-  %49 = icmp sgt i32 %47, %48
-  %50 = select i1 %49, i32 %47, i32 %48
-  %51 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 17), align 4
-  %52 = icmp sgt i32 %50, %51
-  %53 = select i1 %52, i32 %50, i32 %51
-  %54 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 18), align 8
-  %55 = icmp sgt i32 %53, %54
-  %56 = select i1 %55, i32 %53, i32 %54
-  %57 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 19), align 4
-  %58 = icmp sgt i32 %56, %57
-  %59 = select i1 %58, i32 %56, i32 %57
-  %60 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 20), align 16
-  %61 = icmp sgt i32 %59, %60
-  %62 = select i1 %61, i32 %59, i32 %60
-  %63 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 21), align 4
-  %64 = icmp sgt i32 %62, %63
-  %65 = select i1 %64, i32 %62, i32 %63
-  %66 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 22), align 8
-  %67 = icmp sgt i32 %65, %66
-  %68 = select i1 %67, i32 %65, i32 %66
-  %69 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 23), align 4
-  %70 = icmp sgt i32 %68, %69
-  %71 = select i1 %70, i32 %68, i32 %69
-  %72 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 24), align 16
-  %73 = icmp sgt i32 %71, %72
-  %74 = select i1 %73, i32 %71, i32 %72
-  %75 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 25), align 4
-  %76 = icmp sgt i32 %74, %75
-  %77 = select i1 %76, i32 %74, i32 %75
-  %78 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 26), align 8
-  %79 = icmp sgt i32 %77, %78
-  %80 = select i1 %79, i32 %77, i32 %78
-  %81 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 27), align 4
-  %82 = icmp sgt i32 %80, %81
-  %83 = select i1 %82, i32 %80, i32 %81
-  %84 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 28), align 16
-  %85 = icmp sgt i32 %83, %84
-  %86 = select i1 %85, i32 %83, i32 %84
-  %87 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 29), align 4
-  %88 = icmp sgt i32 %86, %87
-  %89 = select i1 %88, i32 %86, i32 %87
-  %90 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 30), align 8
-  %91 = icmp sgt i32 %89, %90
-  %92 = select i1 %91, i32 %89, i32 %90
-  %93 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 31), align 4
-  %94 = icmp sgt i32 %92, %93
-  %95 = select i1 %94, i32 %92, i32 %93
-  ret i32 %95
+  %3 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 4), align 4
+  %4 = call i32 @llvm.smax.i32(i32 %2, i32 %3)
+  %5 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 8), align 8
+  %6 = call i32 @llvm.smax.i32(i32 %4, i32 %5)
+  %7 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 12), align 4
+  %8 = call i32 @llvm.smax.i32(i32 %6, i32 %7)
+  %9 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 16), align 16
+  %10 = call i32 @llvm.smax.i32(i32 %8, i32 %9)
+  %11 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 20), align 4
+  %12 = call i32 @llvm.smax.i32(i32 %10, i32 %11)
+  %13 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 24), align 8
+  %14 = call i32 @llvm.smax.i32(i32 %12, i32 %13)
+  %15 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 28), align 4
+  %16 = call i32 @llvm.smax.i32(i32 %14, i32 %15)
+  %17 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 32), align 16
+  %18 = call i32 @llvm.smax.i32(i32 %16, i32 %17)
+  %19 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 36), align 4
+  %20 = call i32 @llvm.smax.i32(i32 %18, i32 %19)
+  %21 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 40), align 8
+  %22 = call i32 @llvm.smax.i32(i32 %20, i32 %21)
+  %23 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 44), align 4
+  %24 = call i32 @llvm.smax.i32(i32 %22, i32 %23)
+  %25 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 48), align 16
+  %26 = call i32 @llvm.smax.i32(i32 %24, i32 %25)
+  %27 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 52), align 4
+  %28 = call i32 @llvm.smax.i32(i32 %26, i32 %27)
+  %29 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 56), align 8
+  %30 = call i32 @llvm.smax.i32(i32 %28, i32 %29)
+  %31 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 60), align 4
+  %32 = call i32 @llvm.smax.i32(i32 %30, i32 %31)
+  %33 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 64), align 16
+  %34 = call i32 @llvm.smax.i32(i32 %32, i32 %33)
+  %35 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 68), align 4
+  %36 = call i32 @llvm.smax.i32(i32 %34, i32 %35)
+  %37 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 72), align 8
+  %38 = call i32 @llvm.smax.i32(i32 %36, i32 %37)
+  %39 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 76), align 4
+  %40 = call i32 @llvm.smax.i32(i32 %38, i32 %39)
+  %41 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 80), align 16
+  %42 = call i32 @llvm.smax.i32(i32 %40, i32 %41)
+  %43 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 84), align 4
+  %44 = call i32 @llvm.smax.i32(i32 %42, i32 %43)
+  %45 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 88), align 8
+  %46 = call i32 @llvm.smax.i32(i32 %44, i32 %45)
+  %47 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 92), align 4
+  %48 = call i32 @llvm.smax.i32(i32 %46, i32 %47)
+  %49 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 96), align 16
+  %50 = call i32 @llvm.smax.i32(i32 %48, i32 %49)
+  %51 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 100), align 4
+  %52 = call i32 @llvm.smax.i32(i32 %50, i32 %51)
+  %53 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 104), align 8
+  %54 = call i32 @llvm.smax.i32(i32 %52, i32 %53)
+  %55 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 108), align 4
+  %56 = call i32 @llvm.smax.i32(i32 %54, i32 %55)
+  %57 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 112), align 16
+  %58 = call i32 @llvm.smax.i32(i32 %56, i32 %57)
+  %59 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 116), align 4
+  %60 = call i32 @llvm.smax.i32(i32 %58, i32 %59)
+  %61 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 120), align 8
+  %62 = call i32 @llvm.smax.i32(i32 %60, i32 %61)
+  %63 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 124), align 4
+  %64 = call i32 @llvm.smax.i32(i32 %62, i32 %63)
+  ret i32 %64
 }
 
 ; Note: legacy test - InstCombine creates maxnum intrinsics for fcmp+select with fastmath on the select.
@@ -764,181 +704,108 @@ define float @maxf32(float) {
   ret float %95
 }
 
-define i32 @maxi8_mutiple_uses(i32) {
-; DEFAULT-LABEL: @maxi8_mutiple_uses(
-; DEFAULT-NEXT:    [[TMP2:%.*]] = load i32, ptr @arr, align 16
-; DEFAULT-NEXT:    [[TMP3:%.*]] = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 1), align 4
-; DEFAULT-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
-; DEFAULT-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
-; DEFAULT-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 2), align 8
-; DEFAULT-NEXT:    [[TMP7:%.*]] = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 6), align 8
-; DEFAULT-NEXT:    [[TMP8:%.*]] = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 7), align 4
-; DEFAULT-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP6]])
-; DEFAULT-NEXT:    [[OP_RDX:%.*]] = icmp sgt i32 [[TMP9]], [[TMP7]]
-; DEFAULT-NEXT:    [[OP_RDX1:%.*]] = select i1 [[OP_RDX]], i32 [[TMP9]], i32 [[TMP7]]
-; DEFAULT-NEXT:    [[OP_RDX2:%.*]] = icmp sgt i32 [[TMP8]], [[TMP5]]
-; DEFAULT-NEXT:    [[OP_RDX3:%.*]] = select i1 [[OP_RDX2]], i32 [[TMP8]], i32 [[TMP5]]
-; DEFAULT-NEXT:    [[OP_RDX4:%.*]] = icmp sgt i32 [[OP_RDX1]], [[OP_RDX3]]
-; DEFAULT-NEXT:    [[OP_RDX5:%.*]] = select i1 [[OP_RDX4]], i32 [[OP_RDX1]], i32 [[OP_RDX3]]
-; DEFAULT-NEXT:    [[TMP10:%.*]] = select i1 [[TMP4]], i32 3, i32 4
-; DEFAULT-NEXT:    store i32 [[TMP10]], ptr @var, align 8
-; DEFAULT-NEXT:    ret i32 [[OP_RDX5]]
-;
-; THRESH-LABEL: @maxi8_mutiple_uses(
-; THRESH-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @arr, align 16
-; THRESH-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0
-; THRESH-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP2]], i32 1
-; THRESH-NEXT:    [[TMP5:%.*]] = icmp sgt i32 [[TMP3]], [[TMP4]]
-; THRESH-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP3]], i32 [[TMP4]]
-; THRESH-NEXT:    [[TMP7:%.*]] = load <4 x i32>, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 2), align 8
-; THRESH-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP7]])
-; THRESH-NEXT:    [[TMP9:%.*]] = load <2 x i32>, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 6), align 8
-; THRESH-NEXT:    [[TMP10:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP8]], i32 0
-; THRESH-NEXT:    [[TMP11:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP6]], i32 1
-; THRESH-NEXT:    [[TMP12:%.*]] = icmp sgt <2 x i32> [[TMP10]], [[TMP11]]
-; THRESH-NEXT:    [[TMP13:%.*]] = select <2 x i1> [[TMP12]], <2 x i32> [[TMP10]], <2 x i32> [[TMP11]]
-; THRESH-NEXT:    [[TMP14:%.*]] = extractelement <2 x i32> [[TMP13]], i32 0
-; THRESH-NEXT:    [[TMP15:%.*]] = extractelement <2 x i32> [[TMP13]], i32 1
-; THRESH-NEXT:    [[OP_RDX4:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
-; THRESH-NEXT:    [[OP_RDX5:%.*]] = select i1 [[OP_RDX4]], i32 [[TMP14]], i32 [[TMP15]]
-; THRESH-NEXT:    [[TMP16:%.*]] = select i1 [[TMP5]], i32 3, i32 4
-; THRESH-NEXT:    store i32 [[TMP16]], ptr @var, align 8
-; THRESH-NEXT:    ret i32 [[OP_RDX5]]
+define i32 @maxi8_mutiple_uses(i32 %0) {
+; CHECK-LABEL: @maxi8_mutiple_uses(
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr @arr, align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x i32> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sgt i32 [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> [[TMP2]])
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP5]], i32 3, i32 4
+; CHECK-NEXT:    store i32 [[TMP7]], ptr @var, align 8
+; CHECK-NEXT:    ret i32 [[TMP6]]
 ;
   %2 = load i32, ptr @arr, align 16
-  %3 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 1), align 4
+  %3 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 4), align 4
   %4 = icmp sgt i32 %2, %3
-  %5 = select i1 %4, i32 %2, i32 %3
-  %6 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 2), align 8
-  %7 = icmp sgt i32 %5, %6
-  %8 = select i1 %7, i32 %5, i32 %6
-  %9 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 3), align 4
-  %10 = icmp sgt i32 %8, %9
-  %11 = select i1 %10, i32 %8, i32 %9
-  %12 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 4), align 16
-  %13 = icmp sgt i32 %11, %12
-  %14 = select i1 %13, i32 %11, i32 %12
-  %15 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 5), align 4
-  %16 = icmp sgt i32 %14, %15
-  %17 = select i1 %16, i32 %14, i32 %15
-  %18 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 6), align 8
-  %19 = icmp sgt i32 %17, %18
-  %20 = select i1 %19, i32 %17, i32 %18
-  %21 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 7), align 4
-  %22 = icmp sgt i32 %20, %21
-  %23 = select i1 %22, i32 %20, i32 %21
-  %24 = select i1 %4, i32 3, i32 4
-  store i32 %24, ptr @var, align 8
-  ret i32 %23
+  %5 = call i32 @llvm.smax.i32(i32 %2, i32 %3)
+  %6 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 8), align 8
+  %7 = call i32 @llvm.smax.i32(i32 %5, i32 %6)
+  %8 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 12), align 4
+  %9 = call i32 @llvm.smax.i32(i32 %7, i32 %8)
+  %10 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 16), align 16
+  %11 = call i32 @llvm.smax.i32(i32 %9, i32 %10)
+  %12 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 20), align 4
+  %13 = call i32 @llvm.smax.i32(i32 %11, i32 %12)
+  %14 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 24), align 8
+  %15 = call i32 @llvm.smax.i32(i32 %13, i32 %14)
+  %16 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 28), align 4
+  %17 = call i32 @llvm.smax.i32(i32 %15, i32 %16)
+  %18 = select i1 %4, i32 3, i32 4
+  store i32 %18, ptr @var, align 8
+  ret i32 %17
 }
 
-define i32 @maxi8_mutiple_uses2(i32) {
-; DEFAULT-LABEL: @maxi8_mutiple_uses2(
-; DEFAULT-NEXT:    [[TMP2:%.*]] = load i32, ptr @arr, align 16
-; DEFAULT-NEXT:    [[TMP3:%.*]] = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 1), align 4
-; DEFAULT-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
-; DEFAULT-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
-; DEFAULT-NEXT:    [[TMP6:%.*]] = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 2), align 8
-; DEFAULT-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], [[TMP6]]
-; DEFAULT-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 [[TMP6]]
-; DEFAULT-NEXT:    [[TMP9:%.*]] = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 3), align 4
-; DEFAULT-NEXT:    [[TMP10:%.*]] = icmp sgt i32 [[TMP8]], [[TMP9]]
-; DEFAULT-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i32 [[TMP8]], i32 [[TMP9]]
-; DEFAULT-NEXT:    [[TMP12:%.*]] = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 4), align 16
-; DEFAULT-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP11]], [[TMP12]]
-; DEFAULT-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
-; DEFAULT-NEXT:    [[TMP15:%.*]] = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 5), align 4
-; DEFAULT-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
-; DEFAULT-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
-; DEFAULT-NEXT:    [[TMP18:%.*]] = select i1 [[TMP10]], i32 3, i32 4
-; DEFAULT-NEXT:    store i32 [[TMP18]], ptr @var, align 8
-; DEFAULT-NEXT:    ret i32 [[TMP17]]
-;
-; THRESH-LABEL: @maxi8_mutiple_uses2(
-; THRESH-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @arr, align 16
-; THRESH-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[TMP2]], i32 0
-; THRESH-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP2]], i32 1
-; THRESH-NEXT:    [[TMP5:%.*]] = icmp sgt i32 [[TMP3]], [[TMP4]]
-; THRESH-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP3]], i32 [[TMP4]]
-; THRESH-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP2]], i32 2
-; THRESH-NEXT:    [[TMP8:%.*]] = icmp sgt i32 [[TMP6]], [[TMP7]]
-; THRESH-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i32 [[TMP6]], i32 [[TMP7]]
-; THRESH-NEXT:    [[TMP10:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3
-; THRESH-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP9]], [[TMP10]]
-; THRESH-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP9]], i32 [[TMP10]]
-; THRESH-NEXT:    [[TMP13:%.*]] = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 4), align 16
-; THRESH-NEXT:    [[TMP14:%.*]] = icmp sgt i32 [[TMP12]], [[TMP13]]
-; THRESH-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[TMP12]], i32 [[TMP13]]
-; THRESH-NEXT:    [[TMP16:%.*]] = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 5), align 4
-; THRESH-NEXT:    [[TMP17:%.*]] = icmp sgt i32 [[TMP15]], [[TMP16]]
-; THRESH-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], i32 [[TMP15]], i32 [[TMP16]]
-; THRESH-NEXT:    [[TMP19:%.*]] = select i1 [[TMP11]], i32 3, i32 4
-; THRESH-NEXT:    store i32 [[TMP19]], ptr @var, align 8
-; THRESH-NEXT:    ret i32 [[TMP18]]
+define i32 @maxi8_mutiple_uses2(i32 %0) {
+; CHECK-LABEL: @maxi8_mutiple_uses2(
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr @arr, align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 4), align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP2]], i32 [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 8), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP4]], i32 [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 12), align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp sgt i32 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP6]], i32 [[TMP7]])
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 16), align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP9]], i32 [[TMP10]])
+; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 20), align 4
+; CHECK-NEXT:    [[TMP13:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP11]], i32 [[TMP12]])
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP8]], i32 3, i32 4
+; CHECK-NEXT:    store i32 [[TMP14]], ptr @var, align 8
+; CHECK-NEXT:    ret i32 [[TMP13]]
 ;
   %2 = load i32, ptr @arr, align 16
-  %3 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 1), align 4
-  %4 = icmp sgt i32 %2, %3
-  %5 = select i1 %4, i32 %2, i32 %3
-  %6 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 2), align 8
-  %7 = icmp sgt i32 %5, %6
-  %8 = select i1 %7, i32 %5, i32 %6
-  %9 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 3), align 4
-  %10 = icmp sgt i32 %8, %9
-  %11 = select i1 %10, i32 %8, i32 %9
-  %12 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 4), align 16
-  %13 = icmp sgt i32 %11, %12
-  %14 = select i1 %13, i32 %11, i32 %12
-  %15 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 5), align 4
-  %16 = icmp sgt i32 %14, %15
-  %17 = select i1 %16, i32 %14, i32 %15
-  %18 = select i1 %10, i32 3, i32 4
-  store i32 %18, ptr @var, align 8
-  ret i32 %17
+  %3 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 4), align 4
+  %4 = call i32 @llvm.smax.i32(i32 %2, i32 %3)
+  %5 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 8), align 8
+  %6 = call i32 @llvm.smax.i32(i32 %4, i32 %5)
+  %7 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 12), align 4
+  %8 = icmp sgt i32 %6, %7
+  %9 = call i32 @llvm.smax.i32(i32 %6, i32 %7)
+  %10 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 16), align 16
+  %11 = call i32 @llvm.smax.i32(i32 %9, i32 %10)
+  %12 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 20), align 4
+  %13 = call i32 @llvm.smax.i32(i32 %11, i32 %12)
+  %14 = select i1 %8, i32 3, i32 4
+  store i32 %14, ptr @var, align 8
+  ret i32 %13
 }
 
-define i32 @maxi8_wrong_parent(i32) {
+define i32 @maxi8_wrong_parent(i32 %0) {
 ; CHECK-LABEL: @maxi8_wrong_parent(
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @arr, align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = load <2 x i32>, ptr @arr, align 16
 ; CHECK-NEXT:    br label [[PP:%.*]]
 ; CHECK:       pp:
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 2), align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i32>, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 6), align 8
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i32>, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 8), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i32>, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 24), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> [[TMP10]])
-; CHECK-NEXT:    ret i32 [[TMP11]]
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP10]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
   %2 = load i32, ptr @arr, align 16
-  %3 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 1), align 4
-  %4 = icmp sgt i32 %2, %3
+  %3 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 4), align 4
   br label %pp
 
 pp:
-  %5 = select i1 %4, i32 %2, i32 %3
-  %6 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 2), align 8
-  %7 = icmp sgt i32 %5, %6
-  %8 = select i1 %7, i32 %5, i32 %6
-  %9 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 3), align 4
-  %10 = icmp sgt i32 %8, %9
-  %11 = select i1 %10, i32 %8, i32 %9
-  %12 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 4), align 16
-  %13 = icmp sgt i32 %11, %12
-  %14 = select i1 %13, i32 %11, i32 %12
-  %15 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 5), align 4
-  %16 = icmp sgt i32 %14, %15
-  %17 = select i1 %16, i32 %14, i32 %15
-  %18 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 6), align 8
-  %19 = icmp sgt i32 %17, %18
-  %20 = select i1 %19, i32 %17, i32 %18
-  %21 = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 7), align 4
-  %22 = icmp sgt i32 %20, %21
-  %23 = select i1 %22, i32 %20, i32 %21
-  ret i32 %23
+  %4 = call i32 @llvm.smax.i32(i32 %2, i32 %3)
+  %5 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 8), align 8
+  %6 = call i32 @llvm.smax.i32(i32 %4, i32 %5)
+  %7 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 12), align 4
+  %8 = call i32 @llvm.smax.i32(i32 %6, i32 %7)
+  %9 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 16), align 16
+  %10 = call i32 @llvm.smax.i32(i32 %8, i32 %9)
+  %11 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 20), align 4
+  %12 = call i32 @llvm.smax.i32(i32 %10, i32 %11)
+  %13 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 24), align 8
+  %14 = call i32 @llvm.smax.i32(i32 %12, i32 %13)
+  %15 = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 28), align 4
+  %16 = call i32 @llvm.smax.i32(i32 %14, i32 %15)
+  ret i32 %16
 }
 
 ; PR38191 - We don't handle array-of-pointer reductions.
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-ctpop.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-ctpop.ll
new file mode 100644
index 0000000000000..67fd3f4acacaa
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-ctpop.ll
@@ -0,0 +1,166 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux -mcpu=skylake-avx512 < %s | FileCheck %s
+
+define i32 @or_nonzero(ptr %p) {
+; CHECK-LABEL: define i32 @or_nonzero(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[INPUT]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[RESULT:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+  %input = load <8 x i8>, ptr %p, align 1
+  %a0 = extractelement <8 x i8> %input, i64 0
+  %a1 = extractelement <8 x i8> %input, i64 1
+  %a2 = extractelement <8 x i8> %input, i64 2
+  %a3 = extractelement <8 x i8> %input, i64 3
+  %a4 = extractelement <8 x i8> %input, i64 4
+  %a5 = extractelement <8 x i8> %input, i64 5
+  %a6 = extractelement <8 x i8> %input, i64 6
+  %a7 = extractelement <8 x i8> %input, i64 7
+  %or1 = or i8 %a0, %a1
+  %or2 = or i8 %or1, %a2
+  %or3 = or i8 %or2, %a3
+  %or4 = or i8 %or3, %a4
+  %or5 = or i8 %or4, %a5
+  %or6 = or i8 %or5, %a6
+  %or7 = or i8 %or6, %a7
+  %cmp = icmp ne i8 %or7, 0
+  %result = zext i1 %cmp to i32
+  ret i32 %result
+}
+
+define i32 @or_nonzero_commuted(ptr %p) {
+; CHECK-LABEL: define i32 @or_nonzero_commuted(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[INPUT]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 0, [[TMP1]]
+; CHECK-NEXT:    [[RESULT:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+  %input = load <8 x i8>, ptr %p, align 1
+  %a0 = extractelement <8 x i8> %input, i64 0
+  %a1 = extractelement <8 x i8> %input, i64 1
+  %a2 = extractelement <8 x i8> %input, i64 2
+  %a3 = extractelement <8 x i8> %input, i64 3
+  %a4 = extractelement <8 x i8> %input, i64 4
+  %a5 = extractelement <8 x i8> %input, i64 5
+  %a6 = extractelement <8 x i8> %input, i64 6
+  %a7 = extractelement <8 x i8> %input, i64 7
+  %or1 = or i8 %a0, %a1
+  %or2 = or i8 %or1, %a2
+  %or3 = or i8 %or2, %a3
+  %or4 = or i8 %or3, %a4
+  %or5 = or i8 %or4, %a5
+  %or6 = or i8 %or5, %a6
+  %or7 = or i8 %or6, %a7
+  %cmp = icmp ne i8 0, %or7
+  %result = zext i1 %cmp to i32
+  ret i32 %result
+}
+
+define i32 @or_zero(ptr %p) {
+; CHECK-LABEL: define i32 @or_zero(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[INPUT]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[TMP3]], 0
+; CHECK-NEXT:    [[RESULT:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+  %input = load <8 x i8>, ptr %p, align 1
+  %a0 = extractelement <8 x i8> %input, i64 0
+  %a1 = extractelement <8 x i8> %input, i64 1
+  %a2 = extractelement <8 x i8> %input, i64 2
+  %a3 = extractelement <8 x i8> %input, i64 3
+  %a4 = extractelement <8 x i8> %input, i64 4
+  %a5 = extractelement <8 x i8> %input, i64 5
+  %a6 = extractelement <8 x i8> %input, i64 6
+  %a7 = extractelement <8 x i8> %input, i64 7
+  %or1 = or i8 %a0, %a1
+  %or2 = or i8 %or1, %a2
+  %or3 = or i8 %or2, %a3
+  %or4 = or i8 %or3, %a4
+  %or5 = or i8 %or4, %a5
+  %or6 = or i8 %or5, %a6
+  %or7 = or i8 %or6, %a7
+  %cmp = icmp eq i8 %or7, 0
+  %result = zext i1 %cmp to i32
+  ret i32 %result
+}
+
+define i32 @umax_nonzero(ptr %p) {
+; CHECK-LABEL: define i32 @umax_nonzero(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> [[INPUT]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[RESULT:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+  %input = load <8 x i8>, ptr %p, align 1
+  %a0 = extractelement <8 x i8> %input, i64 0
+  %a1 = extractelement <8 x i8> %input, i64 1
+  %a2 = extractelement <8 x i8> %input, i64 2
+  %a3 = extractelement <8 x i8> %input, i64 3
+  %a4 = extractelement <8 x i8> %input, i64 4
+  %a5 = extractelement <8 x i8> %input, i64 5
+  %a6 = extractelement <8 x i8> %input, i64 6
+  %a7 = extractelement <8 x i8> %input, i64 7
+  %c1 = icmp ugt i8 %a0, %a1
+  %m1 = select i1 %c1, i8 %a0, i8 %a1
+  %c2 = icmp ugt i8 %m1, %a2
+  %m2 = select i1 %c2, i8 %m1, i8 %a2
+  %c3 = icmp ugt i8 %m2, %a3
+  %m3 = select i1 %c3, i8 %m2, i8 %a3
+  %c4 = icmp ugt i8 %m3, %a4
+  %m4 = select i1 %c4, i8 %m3, i8 %a4
+  %c5 = icmp ugt i8 %m4, %a5
+  %m5 = select i1 %c5, i8 %m4, i8 %a5
+  %c6 = icmp ugt i8 %m5, %a6
+  %m6 = select i1 %c6, i8 %m5, i8 %a6
+  %c7 = icmp ugt i8 %m6, %a7
+  %m7 = select i1 %c7, i8 %m6, i8 %a7
+  %cmp = icmp ne i8 %m7, 0
+  %result = zext i1 %cmp to i32
+  ret i32 %result
+}
+
+define i32 @umax_zero_commuted(ptr %p) {
+; CHECK-LABEL: define i32 @umax_zero_commuted(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> [[INPUT]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 0, [[TMP1]]
+; CHECK-NEXT:    [[RESULT:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+  %input = load <8 x i8>, ptr %p, align 1
+  %a0 = extractelement <8 x i8> %input, i64 0
+  %a1 = extractelement <8 x i8> %input, i64 1
+  %a2 = extractelement <8 x i8> %input, i64 2
+  %a3 = extractelement <8 x i8> %input, i64 3
+  %a4 = extractelement <8 x i8> %input, i64 4
+  %a5 = extractelement <8 x i8> %input, i64 5
+  %a6 = extractelement <8 x i8> %input, i64 6
+  %a7 = extractelement <8 x i8> %input, i64 7
+  %c1 = icmp ugt i8 %a0, %a1
+  %m1 = select i1 %c1, i8 %a0, i8 %a1
+  %c2 = icmp ugt i8 %m1, %a2
+  %m2 = select i1 %c2, i8 %m1, i8 %a2
+  %c3 = icmp ugt i8 %m2, %a3
+  %m3 = select i1 %c3, i8 %m2, i8 %a3
+  %c4 = icmp ugt i8 %m3, %a4
+  %m4 = select i1 %c4, i8 %m3, i8 %a4
+  %c5 = icmp ugt i8 %m4, %a5
+  %m5 = select i1 %c5, i8 %m4, i8 %a5
+  %c6 = icmp ugt i8 %m5, %a6
+  %m6 = select i1 %c6, i8 %m5, i8 %a6
+  %c7 = icmp ugt i8 %m6, %a7
+  %m7 = select i1 %c7, i8 %m6, i8 %a7
+  %cmp = icmp eq i8 0, %m7
+  %result = zext i1 %cmp to i32
+  ret i32 %result
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction.ll
index 35595b79cd708..2e7829a22613e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction.ll
@@ -75,16 +75,15 @@ define i32 @horiz_max_multiple_uses(ptr %x, ptr %p) {
 ; CHECK-NEXT:    [[X4:%.*]] = getelementptr [32 x i32], ptr [[X:%.*]], i64 0, i64 4
 ; CHECK-NEXT:    [[X5:%.*]] = getelementptr [32 x i32], ptr [[X]], i64 0, i64 5
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[X]], align 4
-; CHECK-NEXT:    [[T4:%.*]] = load i32, ptr [[X4]], align 4
-; CHECK-NEXT:    [[T5:%.*]] = load i32, ptr [[X5]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP1]])
-; CHECK-NEXT:    [[MAX_ROOT_CMP:%.*]] = icmp sgt i32 [[TMP2]], [[T4]]
-; CHECK-NEXT:    [[MAX_ROOT_SEL:%.*]] = select i1 [[MAX_ROOT_CMP]], i32 [[TMP2]], i32 [[T4]]
-; CHECK-NEXT:    [[C012345:%.*]] = icmp sgt i32 [[MAX_ROOT_SEL]], [[T5]]
-; CHECK-NEXT:    [[T17:%.*]] = select i1 [[C012345]], i32 [[MAX_ROOT_SEL]], i32 [[T5]]
-; CHECK-NEXT:    [[THREE_OR_FOUR:%.*]] = select i1 [[MAX_ROOT_CMP]], i32 3, i32 4
+; CHECK-NEXT:    [[T6:%.*]] = load i32, ptr [[X4]], align 4
+; CHECK-NEXT:    [[T7:%.*]] = load i32, ptr [[X5]], align 4
+; CHECK-NEXT:    [[RDX4:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT:    [[MAX_ROOT_CMP1:%.*]] = icmp sgt i32 [[RDX4]], [[T6]]
+; CHECK-NEXT:    [[MAX_ROOT_SEL:%.*]] = select i1 [[MAX_ROOT_CMP1]], i32 [[RDX4]], i32 [[T6]]
+; CHECK-NEXT:    [[T18:%.*]] = call i32 @llvm.smax.i32(i32 [[MAX_ROOT_SEL]], i32 [[T7]])
+; CHECK-NEXT:    [[THREE_OR_FOUR:%.*]] = select i1 [[MAX_ROOT_CMP1]], i32 3, i32 4
 ; CHECK-NEXT:    store i32 [[THREE_OR_FOUR]], ptr [[P:%.*]], align 8
-; CHECK-NEXT:    ret i32 [[T17]]
+; CHECK-NEXT:    ret i32 [[T18]]
 ;
   %x1 = getelementptr [32 x i32], ptr %x, i64 0, i64 1
   %x2 = getelementptr [32 x i32], ptr %x, i64 0, i64 2
@@ -99,16 +98,12 @@ define i32 @horiz_max_multiple_uses(ptr %x, ptr %p) {
   %t4 = load i32, ptr %x4
   %t5 = load i32, ptr %x5
 
-  %c01 = icmp sgt i32 %t0, %t1
-  %s5 = select i1 %c01, i32 %t0, i32 %t1
-  %c012 = icmp sgt i32 %s5, %t2
-  %t8 = select i1 %c012, i32 %s5, i32 %t2
-  %c0123 = icmp sgt i32 %t8, %t3
-  %rdx4 = select i1 %c0123, i32 %t8, i32 %t3
+  %s5 = call i32 @llvm.smax(i32 %t0, i32 %t1)
+  %t8 = call i32 @llvm.smax(i32 %s5, i32 %t2)
+  %rdx4 = call i32 @llvm.smax(i32 %t8, i32 %t3)
   %MAX_ROOT_CMP = icmp sgt i32 %rdx4, %t4
   %MAX_ROOT_SEL = select i1 %MAX_ROOT_CMP, i32 %rdx4, i32 %t4
-  %c012345 = icmp sgt i32 %MAX_ROOT_SEL, %t5
-  %t17 = select i1 %c012345, i32 %MAX_ROOT_SEL, i32 %t5
+  %t17 = call i32 @llvm.smax(i32 %MAX_ROOT_SEL, i32 %t5)
   %three_or_four = select i1 %MAX_ROOT_CMP, i32 3, i32 4
   store i32 %three_or_four, ptr %p, align 8
   ret i32 %t17
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/undef_vect.ll b/llvm/test/Transforms/SLPVectorizer/X86/undef_vect.ll
index 583b992ff392a..9c00c368bf0bd 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/undef_vect.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/undef_vect.ll
@@ -7,10 +7,9 @@ define void @_Z2azv(ptr %p) local_unnamed_addr {
 ; CHECK-LABEL: @_Z2azv(
 ; CHECK-NEXT:  for.body.lr.ph:
 ; CHECK-NEXT:    [[DOTSROA_CAST_4:%.*]] = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", ptr [[P:%.*]], i64 4, i32 0
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr [[DOTSROA_CAST_4]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> [[TMP1]])
-; CHECK-NEXT:    [[OP_RDX:%.*]] = icmp sgt i32 0, [[TMP2]]
-; CHECK-NEXT:    [[OP_RDX1:%.*]] = select i1 [[OP_RDX]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i32>, ptr [[DOTSROA_CAST_4]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> [[TMP0]])
+; CHECK-NEXT:    [[OP_RDX1:%.*]] = call i32 @llvm.smax.i32(i32 0, i32 [[TMP2]])
 ; CHECK-NEXT:    [[DOTSROA_SPECULATED_9:%.*]] = select i1 false, i32 0, i32 [[OP_RDX1]]
 ; CHECK-NEXT:    [[CMP_I1_10:%.*]] = icmp slt i32 [[DOTSROA_SPECULATED_9]], 0
 ; CHECK-NEXT:    ret void
@@ -20,36 +19,27 @@ for.body.lr.ph:
   %retval.sroa.0.0.copyload.i5.4 = load i32, ptr %.sroa_cast.4, align 4
   %.sroa_raw_idx.4 = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", ptr %p, i64 4, i32 1
   %retval.sroa.0.0.copyload.i7.4 = load i32, ptr %.sroa_raw_idx.4, align 4
-  %cmp.i2.4 = icmp slt i32 %retval.sroa.0.0.copyload.i5.4, %retval.sroa.0.0.copyload.i7.4
-  %0 = select i1 %cmp.i2.4, i32 %retval.sroa.0.0.copyload.i7.4, i32 %retval.sroa.0.0.copyload.i5.4
-  %cmp.i1.4 = icmp slt i32 0, %0
-  %.sroa.speculated.4 = select i1 %cmp.i1.4, i32 %0, i32 0
+  %0 = call i32 @llvm.smax(i32 %retval.sroa.0.0.copyload.i7.4, i32 %retval.sroa.0.0.copyload.i5.4)
+  %.sroa.speculated.4 = call i32 @llvm.smax(i32 %0, i32 0)
   %.sroa_cast.5 = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", ptr %p, i64 5, i32 0
   %retval.sroa.0.0.copyload.i5.5 = load i32, ptr %.sroa_cast.5, align 4
   %.sroa_raw_idx.5 = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", ptr %p, i64 5, i32 1
   %retval.sroa.0.0.copyload.i7.5 = load i32, ptr %.sroa_raw_idx.5, align 4
-  %cmp.i2.5 = icmp slt i32 %retval.sroa.0.0.copyload.i5.5, %retval.sroa.0.0.copyload.i7.5
-  %1 = select i1 %cmp.i2.5, i32 %retval.sroa.0.0.copyload.i7.5, i32 %retval.sroa.0.0.copyload.i5.5
-  %cmp.i1.5 = icmp slt i32 %.sroa.speculated.4, %1
-  %.sroa.speculated.5 = select i1 %cmp.i1.5, i32 %1, i32 %.sroa.speculated.4
+  %1 = call i32 @llvm.smax(i32 %retval.sroa.0.0.copyload.i7.5, i32 %retval.sroa.0.0.copyload.i5.5)
+  %.sroa.speculated.5 = call i32 @llvm.smax(i32 %1, i32 %.sroa.speculated.4)
   %.sroa_cast.6 = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", ptr %p, i64 6, i32 0
   %retval.sroa.0.0.copyload.i5.6 = load i32, ptr %.sroa_cast.6, align 4
   %.sroa_raw_idx.6 = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", ptr %p, i64 6, i32 1
   %retval.sroa.0.0.copyload.i7.6 = load i32, ptr %.sroa_raw_idx.6, align 4
-  %cmp.i2.6 = icmp slt i32 %retval.sroa.0.0.copyload.i5.6, %retval.sroa.0.0.copyload.i7.6
-  %2 = select i1 %cmp.i2.6, i32 %retval.sroa.0.0.copyload.i7.6, i32 %retval.sroa.0.0.copyload.i5.6
-  %cmp.i1.6 = icmp slt i32 %.sroa.speculated.5, %2
-  %.sroa.speculated.6 = select i1 %cmp.i1.6, i32 %2, i32 %.sroa.speculated.5
+  %2 = call i32 @llvm.smax(i32 %retval.sroa.0.0.copyload.i7.6, i32 %retval.sroa.0.0.copyload.i5.6)
+  %.sroa.speculated.6 = call i32 @llvm.smax(i32 %2, i32 %.sroa.speculated.5)
   %.sroa_cast.7 = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", ptr %p, i64 7, i32 0
   %retval.sroa.0.0.copyload.i5.7 = load i32, ptr %.sroa_cast.7, align 4
   %.sroa_raw_idx.7 = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", ptr %p, i64 7, i32 1
   %retval.sroa.0.0.copyload.i7.7 = load i32, ptr %.sroa_raw_idx.7, align 4
-  %cmp.i2.7 = icmp slt i32 %retval.sroa.0.0.copyload.i5.7, %retval.sroa.0.0.copyload.i7.7
-  %3 = select i1 %cmp.i2.7, i32 %retval.sroa.0.0.copyload.i7.7, i32 %retval.sroa.0.0.copyload.i5.7
-  %cmp.i1.7 = icmp slt i32 %.sroa.speculated.6, %3
-  %.sroa.speculated.7 = select i1 %cmp.i1.7, i32 %3, i32 %.sroa.speculated.6
-  %cmp.i1.8 = icmp slt i32 %.sroa.speculated.7, 0
-  %.sroa.speculated.8 = select i1 %cmp.i1.8, i32 0, i32 %.sroa.speculated.7
+  %3 = call i32 @llvm.smax(i32 %retval.sroa.0.0.copyload.i7.7, i32 %retval.sroa.0.0.copyload.i5.7)
+  %.sroa.speculated.7 = call i32 @llvm.smax(i32 %3, i32 %.sroa.speculated.6)
+  %.sroa.speculated.8 = call i32 @llvm.smax(i32 0, i32 %.sroa.speculated.7)
   %.sroa.speculated.9 = select i1 0, i32 0, i32 %.sroa.speculated.8
   %cmp.i1.10 = icmp slt i32 %.sroa.speculated.9, 0
   ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/used-reduced-op.ll b/llvm/test/Transforms/SLPVectorizer/X86/used-reduced-op.ll
index eef0e092200b3..e5fba188a5be3 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/used-reduced-op.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/used-reduced-op.ll
@@ -8,136 +8,146 @@
 define void @n() local_unnamed_addr #0 {
 ; CHECK-LABEL: @n(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = load <32 x i32>, ptr @k, align 16
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr @k, align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr getelementptr inbounds ([8 x [4 x i32]], ptr @k, i64 0, i64 1, i64 0), align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr getelementptr inbounds ([8 x [4 x i32]], ptr @k, i64 0, i64 2, i64 0), align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr getelementptr inbounds ([8 x [4 x i32]], ptr @k, i64 0, i64 3, i64 0), align 16
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr getelementptr inbounds ([8 x [4 x i32]], ptr @k, i64 0, i64 4, i64 0), align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr getelementptr inbounds ([8 x [4 x i32]], ptr @k, i64 0, i64 5, i64 0), align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr getelementptr inbounds ([8 x [4 x i32]], ptr @k, i64 0, i64 6, i64 0), align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i32>, ptr getelementptr inbounds ([8 x [4 x i32]], ptr @k, i64 0, i64 7, i64 0), align 16
 ; CHECK-NEXT:    br label [[FOR_COND:%.*]]
 ; CHECK:       for.cond:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_COND]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[B_0:%.*]] = phi i32 [ [[SPEC_SELECT8_3_7:%.*]], [[FOR_COND]] ], [ undef, [[ENTRY]] ]
 ; CHECK-NEXT:    [[TMP29:%.*]] = trunc i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[TMP30:%.*]] = add i32 [[TMP29]], -183
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <32 x i32> poison, i32 [[TMP30]], i32 0
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <32 x i32> [[TMP3]], <32 x i32> poison, <32 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP5:%.*]] = sub <32 x i32> [[TMP4]], [[TMP0]]
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp slt <32 x i32> [[TMP5]], zeroinitializer
-; CHECK-NEXT:    [[TMP7:%.*]] = sub nsw <32 x i32> zeroinitializer, [[TMP5]]
-; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP6]], <32 x i32> [[TMP7]], <32 x i32> [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <32 x i32> [[TMP8]], <32 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP30]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = sub <4 x i32> [[TMP11]], [[TMP0]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> [[TMP12]], i1 true)
 ; CHECK-NEXT:    [[TMP37:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP9]])
-; CHECK-NEXT:    [[OP_RDX:%.*]] = icmp slt i32 [[TMP37]], [[B_0]]
-; CHECK-NEXT:    [[OP_RDX1:%.*]] = select i1 [[OP_RDX]], i32 [[TMP37]], i32 [[B_0]]
-; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <32 x i32> [[TMP8]], i32 4
-; CHECK-NEXT:    [[CMP12_118:%.*]] = icmp slt i32 [[TMP13]], [[OP_RDX1]]
-; CHECK-NEXT:    [[SPEC_SELECT8_120:%.*]] = select i1 [[CMP12_118]], i32 [[TMP13]], i32 [[OP_RDX1]]
-; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <32 x i32> [[TMP8]], i32 5
-; CHECK-NEXT:    [[CMP12_1_1:%.*]] = icmp slt i32 [[TMP41]], [[SPEC_SELECT8_120]]
-; CHECK-NEXT:    [[NARROW:%.*]] = or i1 [[CMP12_1_1]], [[CMP12_118]]
-; CHECK-NEXT:    [[SPEC_SELECT8_1_1:%.*]] = select i1 [[CMP12_1_1]], i32 [[TMP41]], i32 [[SPEC_SELECT8_120]]
-; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <32 x i32> [[TMP8]], i32 6
-; CHECK-NEXT:    [[CMP12_2_1:%.*]] = icmp slt i32 [[TMP43]], [[SPEC_SELECT8_1_1]]
-; CHECK-NEXT:    [[NARROW34:%.*]] = or i1 [[CMP12_2_1]], [[NARROW]]
-; CHECK-NEXT:    [[SPEC_SELECT8_2_1:%.*]] = select i1 [[CMP12_2_1]], i32 [[TMP43]], i32 [[SPEC_SELECT8_1_1]]
-; CHECK-NEXT:    [[TMP45:%.*]] = extractelement <32 x i32> [[TMP8]], i32 7
-; CHECK-NEXT:    [[CMP12_3_1:%.*]] = icmp slt i32 [[TMP45]], [[SPEC_SELECT8_2_1]]
-; CHECK-NEXT:    [[NARROW35:%.*]] = or i1 [[CMP12_3_1]], [[NARROW34]]
+; CHECK-NEXT:    [[TMP15:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP37]], i32 [[B_0]])
+; CHECK-NEXT:    [[TMP16:%.*]] = sub <4 x i32> [[TMP11]], [[TMP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> [[TMP16]], i1 true)
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <4 x i32> [[TMP17]], i32 0
+; CHECK-NEXT:    [[SPEC_SELECT8_120:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP18]], i32 [[TMP15]])
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <4 x i32> [[TMP17]], i32 1
+; CHECK-NEXT:    [[SPEC_SELECT8_1_1:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP19]], i32 [[SPEC_SELECT8_120]])
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x i32> [[TMP17]], i32 2
+; CHECK-NEXT:    [[SPEC_SELECT8_2_1:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP20]], i32 [[SPEC_SELECT8_1_1]])
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <4 x i32> poison, i32 [[TMP15]], i32 0
+; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <4 x i32> [[TMP21]], i32 [[SPEC_SELECT8_120]], i32 1
+; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[SPEC_SELECT8_1_1]], i32 2
+; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[SPEC_SELECT8_2_1]], i32 3
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp slt <4 x i32> [[TMP17]], [[TMP24]]
+; CHECK-NEXT:    [[NARROW35:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP25]])
 ; CHECK-NEXT:    [[SPEC_SELECT_3_1:%.*]] = zext i1 [[NARROW35]] to i32
-; CHECK-NEXT:    [[SPEC_SELECT8_3_1:%.*]] = select i1 [[CMP12_3_1]], i32 [[TMP45]], i32 [[SPEC_SELECT8_2_1]]
-; CHECK-NEXT:    [[TMP47:%.*]] = extractelement <32 x i32> [[TMP8]], i32 8
-; CHECK-NEXT:    [[CMP12_224:%.*]] = icmp slt i32 [[TMP47]], [[SPEC_SELECT8_3_1]]
-; CHECK-NEXT:    [[SPEC_SELECT8_226:%.*]] = select i1 [[CMP12_224]], i32 [[TMP47]], i32 [[SPEC_SELECT8_3_1]]
-; CHECK-NEXT:    [[TMP49:%.*]] = extractelement <32 x i32> [[TMP8]], i32 9
-; CHECK-NEXT:    [[CMP12_1_2:%.*]] = icmp slt i32 [[TMP49]], [[SPEC_SELECT8_226]]
-; CHECK-NEXT:    [[TMP50:%.*]] = or i1 [[CMP12_1_2]], [[CMP12_224]]
-; CHECK-NEXT:    [[SPEC_SELECT8_1_2:%.*]] = select i1 [[CMP12_1_2]], i32 [[TMP49]], i32 [[SPEC_SELECT8_226]]
-; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <32 x i32> [[TMP8]], i32 10
-; CHECK-NEXT:    [[CMP12_2_2:%.*]] = icmp slt i32 [[TMP52]], [[SPEC_SELECT8_1_2]]
-; CHECK-NEXT:    [[TMP53:%.*]] = or i1 [[CMP12_2_2]], [[TMP50]]
-; CHECK-NEXT:    [[SPEC_SELECT8_2_2:%.*]] = select i1 [[CMP12_2_2]], i32 [[TMP52]], i32 [[SPEC_SELECT8_1_2]]
-; CHECK-NEXT:    [[TMP55:%.*]] = extractelement <32 x i32> [[TMP8]], i32 11
-; CHECK-NEXT:    [[CMP12_3_2:%.*]] = icmp slt i32 [[TMP55]], [[SPEC_SELECT8_2_2]]
-; CHECK-NEXT:    [[TMP56:%.*]] = or i1 [[CMP12_3_2]], [[TMP53]]
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <4 x i32> [[TMP17]], i32 3
+; CHECK-NEXT:    [[SPEC_SELECT8_3_1:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP27]], i32 [[SPEC_SELECT8_2_1]])
+; CHECK-NEXT:    [[TMP28:%.*]] = sub <4 x i32> [[TMP11]], [[TMP2]]
+; CHECK-NEXT:    [[TMP38:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> [[TMP28]], i1 true)
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <4 x i32> [[TMP38]], i32 0
+; CHECK-NEXT:    [[SPEC_SELECT8_226:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP50]], i32 [[SPEC_SELECT8_3_1]])
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <4 x i32> [[TMP38]], i32 1
+; CHECK-NEXT:    [[SPEC_SELECT8_1_2:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP31]], i32 [[SPEC_SELECT8_226]])
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <4 x i32> [[TMP38]], i32 2
+; CHECK-NEXT:    [[SPEC_SELECT8_2_2:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP32]], i32 [[SPEC_SELECT8_1_2]])
+; CHECK-NEXT:    [[TMP33:%.*]] = insertelement <4 x i32> poison, i32 [[SPEC_SELECT8_3_1]], i32 0
+; CHECK-NEXT:    [[TMP34:%.*]] = insertelement <4 x i32> [[TMP33]], i32 [[SPEC_SELECT8_226]], i32 1
+; CHECK-NEXT:    [[TMP35:%.*]] = insertelement <4 x i32> [[TMP34]], i32 [[SPEC_SELECT8_1_2]], i32 2
+; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <4 x i32> [[TMP35]], i32 [[SPEC_SELECT8_2_2]], i32 3
+; CHECK-NEXT:    [[TMP62:%.*]] = icmp slt <4 x i32> [[TMP38]], [[TMP36]]
+; CHECK-NEXT:    [[TMP56:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP62]])
 ; CHECK-NEXT:    [[SPEC_SELECT_3_2:%.*]] = select i1 [[TMP56]], i32 2, i32 [[SPEC_SELECT_3_1]]
-; CHECK-NEXT:    [[SPEC_SELECT8_3_2:%.*]] = select i1 [[CMP12_3_2]], i32 [[TMP55]], i32 [[SPEC_SELECT8_2_2]]
-; CHECK-NEXT:    [[TMP58:%.*]] = extractelement <32 x i32> [[TMP8]], i32 12
-; CHECK-NEXT:    [[CMP12_330:%.*]] = icmp slt i32 [[TMP58]], [[SPEC_SELECT8_3_2]]
-; CHECK-NEXT:    [[SPEC_SELECT8_332:%.*]] = select i1 [[CMP12_330]], i32 [[TMP58]], i32 [[SPEC_SELECT8_3_2]]
-; CHECK-NEXT:    [[TMP60:%.*]] = extractelement <32 x i32> [[TMP8]], i32 13
-; CHECK-NEXT:    [[CMP12_1_3:%.*]] = icmp slt i32 [[TMP60]], [[SPEC_SELECT8_332]]
-; CHECK-NEXT:    [[TMP61:%.*]] = or i1 [[CMP12_1_3]], [[CMP12_330]]
-; CHECK-NEXT:    [[SPEC_SELECT8_1_3:%.*]] = select i1 [[CMP12_1_3]], i32 [[TMP60]], i32 [[SPEC_SELECT8_332]]
-; CHECK-NEXT:    [[TMP63:%.*]] = extractelement <32 x i32> [[TMP8]], i32 14
-; CHECK-NEXT:    [[CMP12_2_3:%.*]] = icmp slt i32 [[TMP63]], [[SPEC_SELECT8_1_3]]
-; CHECK-NEXT:    [[TMP64:%.*]] = or i1 [[CMP12_2_3]], [[TMP61]]
-; CHECK-NEXT:    [[SPEC_SELECT8_2_3:%.*]] = select i1 [[CMP12_2_3]], i32 [[TMP63]], i32 [[SPEC_SELECT8_1_3]]
-; CHECK-NEXT:    [[TMP66:%.*]] = extractelement <32 x i32> [[TMP8]], i32 15
-; CHECK-NEXT:    [[CMP12_3_3:%.*]] = icmp slt i32 [[TMP66]], [[SPEC_SELECT8_2_3]]
-; CHECK-NEXT:    [[TMP67:%.*]] = or i1 [[CMP12_3_3]], [[TMP64]]
+; CHECK-NEXT:    [[TMP39:%.*]] = extractelement <4 x i32> [[TMP38]], i32 3
+; CHECK-NEXT:    [[SPEC_SELECT8_3_2:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP39]], i32 [[SPEC_SELECT8_2_2]])
+; CHECK-NEXT:    [[TMP40:%.*]] = sub <4 x i32> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[TMP41:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> [[TMP40]], i1 true)
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <4 x i32> [[TMP41]], i32 0
+; CHECK-NEXT:    [[SPEC_SELECT8_332:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP42]], i32 [[SPEC_SELECT8_3_2]])
+; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <4 x i32> [[TMP41]], i32 1
+; CHECK-NEXT:    [[SPEC_SELECT8_1_3:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP43]], i32 [[SPEC_SELECT8_332]])
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <4 x i32> [[TMP41]], i32 2
+; CHECK-NEXT:    [[SPEC_SELECT8_2_3:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP44]], i32 [[SPEC_SELECT8_1_3]])
+; CHECK-NEXT:    [[TMP45:%.*]] = insertelement <4 x i32> poison, i32 [[SPEC_SELECT8_3_2]], i32 0
+; CHECK-NEXT:    [[TMP46:%.*]] = insertelement <4 x i32> [[TMP45]], i32 [[SPEC_SELECT8_332]], i32 1
+; CHECK-NEXT:    [[TMP47:%.*]] = insertelement <4 x i32> [[TMP46]], i32 [[SPEC_SELECT8_1_3]], i32 2
+; CHECK-NEXT:    [[TMP48:%.*]] = insertelement <4 x i32> [[TMP47]], i32 [[SPEC_SELECT8_2_3]], i32 3
+; CHECK-NEXT:    [[TMP49:%.*]] = icmp slt <4 x i32> [[TMP41]], [[TMP48]]
+; CHECK-NEXT:    [[TMP67:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP49]])
 ; CHECK-NEXT:    [[SPEC_SELECT_3_3:%.*]] = select i1 [[TMP67]], i32 3, i32 [[SPEC_SELECT_3_2]]
-; CHECK-NEXT:    [[SPEC_SELECT8_3_3:%.*]] = select i1 [[CMP12_3_3]], i32 [[TMP66]], i32 [[SPEC_SELECT8_2_3]]
-; CHECK-NEXT:    [[TMP69:%.*]] = extractelement <32 x i32> [[TMP8]], i32 16
-; CHECK-NEXT:    [[CMP12_4:%.*]] = icmp slt i32 [[TMP69]], [[SPEC_SELECT8_3_3]]
-; CHECK-NEXT:    [[SPEC_SELECT8_4:%.*]] = select i1 [[CMP12_4]], i32 [[TMP69]], i32 [[SPEC_SELECT8_3_3]]
-; CHECK-NEXT:    [[TMP71:%.*]] = extractelement <32 x i32> [[TMP8]], i32 17
-; CHECK-NEXT:    [[CMP12_1_4:%.*]] = icmp slt i32 [[TMP71]], [[SPEC_SELECT8_4]]
-; CHECK-NEXT:    [[TMP72:%.*]] = or i1 [[CMP12_1_4]], [[CMP12_4]]
-; CHECK-NEXT:    [[SPEC_SELECT8_1_4:%.*]] = select i1 [[CMP12_1_4]], i32 [[TMP71]], i32 [[SPEC_SELECT8_4]]
-; CHECK-NEXT:    [[TMP74:%.*]] = extractelement <32 x i32> [[TMP8]], i32 18
-; CHECK-NEXT:    [[CMP12_2_4:%.*]] = icmp slt i32 [[TMP74]], [[SPEC_SELECT8_1_4]]
-; CHECK-NEXT:    [[TMP75:%.*]] = or i1 [[CMP12_2_4]], [[TMP72]]
-; CHECK-NEXT:    [[SPEC_SELECT8_2_4:%.*]] = select i1 [[CMP12_2_4]], i32 [[TMP74]], i32 [[SPEC_SELECT8_1_4]]
-; CHECK-NEXT:    [[TMP77:%.*]] = extractelement <32 x i32> [[TMP8]], i32 19
-; CHECK-NEXT:    [[CMP12_3_4:%.*]] = icmp slt i32 [[TMP77]], [[SPEC_SELECT8_2_4]]
-; CHECK-NEXT:    [[TMP78:%.*]] = or i1 [[CMP12_3_4]], [[TMP75]]
+; CHECK-NEXT:    [[TMP51:%.*]] = extractelement <4 x i32> [[TMP41]], i32 3
+; CHECK-NEXT:    [[SPEC_SELECT8_3_3:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP51]], i32 [[SPEC_SELECT8_2_3]])
+; CHECK-NEXT:    [[TMP52:%.*]] = sub <4 x i32> [[TMP11]], [[TMP4]]
+; CHECK-NEXT:    [[TMP53:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> [[TMP52]], i1 true)
+; CHECK-NEXT:    [[TMP54:%.*]] = extractelement <4 x i32> [[TMP53]], i32 0
+; CHECK-NEXT:    [[SPEC_SELECT8_4:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP54]], i32 [[SPEC_SELECT8_3_3]])
+; CHECK-NEXT:    [[TMP55:%.*]] = extractelement <4 x i32> [[TMP53]], i32 1
+; CHECK-NEXT:    [[SPEC_SELECT8_1_4:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP55]], i32 [[SPEC_SELECT8_4]])
+; CHECK-NEXT:    [[TMP74:%.*]] = extractelement <4 x i32> [[TMP53]], i32 2
+; CHECK-NEXT:    [[SPEC_SELECT8_2_4:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP74]], i32 [[SPEC_SELECT8_1_4]])
+; CHECK-NEXT:    [[TMP57:%.*]] = insertelement <4 x i32> poison, i32 [[SPEC_SELECT8_3_3]], i32 0
+; CHECK-NEXT:    [[TMP58:%.*]] = insertelement <4 x i32> [[TMP57]], i32 [[SPEC_SELECT8_4]], i32 1
+; CHECK-NEXT:    [[TMP59:%.*]] = insertelement <4 x i32> [[TMP58]], i32 [[SPEC_SELECT8_1_4]], i32 2
+; CHECK-NEXT:    [[TMP60:%.*]] = insertelement <4 x i32> [[TMP59]], i32 [[SPEC_SELECT8_2_4]], i32 3
+; CHECK-NEXT:    [[TMP61:%.*]] = icmp slt <4 x i32> [[TMP53]], [[TMP60]]
+; CHECK-NEXT:    [[TMP78:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP61]])
 ; CHECK-NEXT:    [[SPEC_SELECT_3_4:%.*]] = select i1 [[TMP78]], i32 4, i32 [[SPEC_SELECT_3_3]]
-; CHECK-NEXT:    [[SPEC_SELECT8_3_4:%.*]] = select i1 [[CMP12_3_4]], i32 [[TMP77]], i32 [[SPEC_SELECT8_2_4]]
-; CHECK-NEXT:    [[TMP80:%.*]] = extractelement <32 x i32> [[TMP8]], i32 20
-; CHECK-NEXT:    [[CMP12_5:%.*]] = icmp slt i32 [[TMP80]], [[SPEC_SELECT8_3_4]]
-; CHECK-NEXT:    [[SPEC_SELECT8_5:%.*]] = select i1 [[CMP12_5]], i32 [[TMP80]], i32 [[SPEC_SELECT8_3_4]]
-; CHECK-NEXT:    [[TMP82:%.*]] = extractelement <32 x i32> [[TMP8]], i32 21
-; CHECK-NEXT:    [[CMP12_1_5:%.*]] = icmp slt i32 [[TMP82]], [[SPEC_SELECT8_5]]
-; CHECK-NEXT:    [[TMP83:%.*]] = or i1 [[CMP12_1_5]], [[CMP12_5]]
-; CHECK-NEXT:    [[SPEC_SELECT8_1_5:%.*]] = select i1 [[CMP12_1_5]], i32 [[TMP82]], i32 [[SPEC_SELECT8_5]]
-; CHECK-NEXT:    [[TMP85:%.*]] = extractelement <32 x i32> [[TMP8]], i32 22
-; CHECK-NEXT:    [[CMP12_2_5:%.*]] = icmp slt i32 [[TMP85]], [[SPEC_SELECT8_1_5]]
-; CHECK-NEXT:    [[TMP86:%.*]] = or i1 [[CMP12_2_5]], [[TMP83]]
-; CHECK-NEXT:    [[SPEC_SELECT8_2_5:%.*]] = select i1 [[CMP12_2_5]], i32 [[TMP85]], i32 [[SPEC_SELECT8_1_5]]
-; CHECK-NEXT:    [[TMP88:%.*]] = extractelement <32 x i32> [[TMP8]], i32 23
-; CHECK-NEXT:    [[CMP12_3_5:%.*]] = icmp slt i32 [[TMP88]], [[SPEC_SELECT8_2_5]]
-; CHECK-NEXT:    [[TMP89:%.*]] = or i1 [[CMP12_3_5]], [[TMP86]]
+; CHECK-NEXT:    [[TMP63:%.*]] = extractelement <4 x i32> [[TMP53]], i32 3
+; CHECK-NEXT:    [[SPEC_SELECT8_3_4:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP63]], i32 [[SPEC_SELECT8_2_4]])
+; CHECK-NEXT:    [[TMP64:%.*]] = sub <4 x i32> [[TMP11]], [[TMP5]]
+; CHECK-NEXT:    [[TMP65:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> [[TMP64]], i1 true)
+; CHECK-NEXT:    [[TMP66:%.*]] = extractelement <4 x i32> [[TMP65]], i32 0
+; CHECK-NEXT:    [[SPEC_SELECT8_5:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP66]], i32 [[SPEC_SELECT8_3_4]])
+; CHECK-NEXT:    [[TMP86:%.*]] = extractelement <4 x i32> [[TMP65]], i32 1
+; CHECK-NEXT:    [[SPEC_SELECT8_1_5:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP86]], i32 [[SPEC_SELECT8_5]])
+; CHECK-NEXT:    [[TMP68:%.*]] = extractelement <4 x i32> [[TMP65]], i32 2
+; CHECK-NEXT:    [[SPEC_SELECT8_2_5:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP68]], i32 [[SPEC_SELECT8_1_5]])
+; CHECK-NEXT:    [[TMP69:%.*]] = insertelement <4 x i32> poison, i32 [[SPEC_SELECT8_3_4]], i32 0
+; CHECK-NEXT:    [[TMP70:%.*]] = insertelement <4 x i32> [[TMP69]], i32 [[SPEC_SELECT8_5]], i32 1
+; CHECK-NEXT:    [[TMP71:%.*]] = insertelement <4 x i32> [[TMP70]], i32 [[SPEC_SELECT8_1_5]], i32 2
+; CHECK-NEXT:    [[TMP72:%.*]] = insertelement <4 x i32> [[TMP71]], i32 [[SPEC_SELECT8_2_5]], i32 3
+; CHECK-NEXT:    [[TMP73:%.*]] = icmp slt <4 x i32> [[TMP65]], [[TMP72]]
+; CHECK-NEXT:    [[TMP89:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP73]])
 ; CHECK-NEXT:    [[SPEC_SELECT_3_5:%.*]] = select i1 [[TMP89]], i32 5, i32 [[SPEC_SELECT_3_4]]
-; CHECK-NEXT:    [[SPEC_SELECT8_3_5:%.*]] = select i1 [[CMP12_3_5]], i32 [[TMP88]], i32 [[SPEC_SELECT8_2_5]]
-; CHECK-NEXT:    [[TMP91:%.*]] = extractelement <32 x i32> [[TMP8]], i32 24
-; CHECK-NEXT:    [[CMP12_6:%.*]] = icmp slt i32 [[TMP91]], [[SPEC_SELECT8_3_5]]
-; CHECK-NEXT:    [[SPEC_SELECT8_6:%.*]] = select i1 [[CMP12_6]], i32 [[TMP91]], i32 [[SPEC_SELECT8_3_5]]
-; CHECK-NEXT:    [[TMP93:%.*]] = extractelement <32 x i32> [[TMP8]], i32 25
-; CHECK-NEXT:    [[CMP12_1_6:%.*]] = icmp slt i32 [[TMP93]], [[SPEC_SELECT8_6]]
-; CHECK-NEXT:    [[TMP94:%.*]] = or i1 [[CMP12_1_6]], [[CMP12_6]]
-; CHECK-NEXT:    [[SPEC_SELECT8_1_6:%.*]] = select i1 [[CMP12_1_6]], i32 [[TMP93]], i32 [[SPEC_SELECT8_6]]
-; CHECK-NEXT:    [[TMP96:%.*]] = extractelement <32 x i32> [[TMP8]], i32 26
-; CHECK-NEXT:    [[CMP12_2_6:%.*]] = icmp slt i32 [[TMP96]], [[SPEC_SELECT8_1_6]]
-; CHECK-NEXT:    [[TMP97:%.*]] = or i1 [[CMP12_2_6]], [[TMP94]]
-; CHECK-NEXT:    [[SPEC_SELECT8_2_6:%.*]] = select i1 [[CMP12_2_6]], i32 [[TMP96]], i32 [[SPEC_SELECT8_1_6]]
-; CHECK-NEXT:    [[TMP99:%.*]] = extractelement <32 x i32> [[TMP8]], i32 27
-; CHECK-NEXT:    [[CMP12_3_6:%.*]] = icmp slt i32 [[TMP99]], [[SPEC_SELECT8_2_6]]
-; CHECK-NEXT:    [[TMP100:%.*]] = or i1 [[CMP12_3_6]], [[TMP97]]
+; CHECK-NEXT:    [[TMP75:%.*]] = extractelement <4 x i32> [[TMP65]], i32 3
+; CHECK-NEXT:    [[SPEC_SELECT8_3_5:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP75]], i32 [[SPEC_SELECT8_2_5]])
+; CHECK-NEXT:    [[TMP76:%.*]] = sub <4 x i32> [[TMP11]], [[TMP6]]
+; CHECK-NEXT:    [[TMP77:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> [[TMP76]], i1 true)
+; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <4 x i32> [[TMP77]], i32 0
+; CHECK-NEXT:    [[SPEC_SELECT8_6:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP98]], i32 [[SPEC_SELECT8_3_5]])
+; CHECK-NEXT:    [[TMP79:%.*]] = extractelement <4 x i32> [[TMP77]], i32 1
+; CHECK-NEXT:    [[SPEC_SELECT8_1_6:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP79]], i32 [[SPEC_SELECT8_6]])
+; CHECK-NEXT:    [[TMP80:%.*]] = extractelement <4 x i32> [[TMP77]], i32 2
+; CHECK-NEXT:    [[SPEC_SELECT8_2_6:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP80]], i32 [[SPEC_SELECT8_1_6]])
+; CHECK-NEXT:    [[TMP81:%.*]] = insertelement <4 x i32> poison, i32 [[SPEC_SELECT8_3_5]], i32 0
+; CHECK-NEXT:    [[TMP82:%.*]] = insertelement <4 x i32> [[TMP81]], i32 [[SPEC_SELECT8_6]], i32 1
+; CHECK-NEXT:    [[TMP83:%.*]] = insertelement <4 x i32> [[TMP82]], i32 [[SPEC_SELECT8_1_6]], i32 2
+; CHECK-NEXT:    [[TMP84:%.*]] = insertelement <4 x i32> [[TMP83]], i32 [[SPEC_SELECT8_2_6]], i32 3
+; CHECK-NEXT:    [[TMP85:%.*]] = icmp slt <4 x i32> [[TMP77]], [[TMP84]]
+; CHECK-NEXT:    [[TMP100:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP85]])
 ; CHECK-NEXT:    [[SPEC_SELECT_3_6:%.*]] = select i1 [[TMP100]], i32 6, i32 [[SPEC_SELECT_3_5]]
-; CHECK-NEXT:    [[SPEC_SELECT8_3_6:%.*]] = select i1 [[CMP12_3_6]], i32 [[TMP99]], i32 [[SPEC_SELECT8_2_6]]
-; CHECK-NEXT:    [[TMP102:%.*]] = extractelement <32 x i32> [[TMP8]], i32 28
-; CHECK-NEXT:    [[CMP12_7:%.*]] = icmp slt i32 [[TMP102]], [[SPEC_SELECT8_3_6]]
-; CHECK-NEXT:    [[SPEC_SELECT8_7:%.*]] = select i1 [[CMP12_7]], i32 [[TMP102]], i32 [[SPEC_SELECT8_3_6]]
-; CHECK-NEXT:    [[TMP104:%.*]] = extractelement <32 x i32> [[TMP8]], i32 29
-; CHECK-NEXT:    [[CMP12_1_7:%.*]] = icmp slt i32 [[TMP104]], [[SPEC_SELECT8_7]]
-; CHECK-NEXT:    [[TMP105:%.*]] = or i1 [[CMP12_1_7]], [[CMP12_7]]
-; CHECK-NEXT:    [[SPEC_SELECT8_1_7:%.*]] = select i1 [[CMP12_1_7]], i32 [[TMP104]], i32 [[SPEC_SELECT8_7]]
-; CHECK-NEXT:    [[TMP107:%.*]] = extractelement <32 x i32> [[TMP8]], i32 30
-; CHECK-NEXT:    [[CMP12_2_7:%.*]] = icmp slt i32 [[TMP107]], [[SPEC_SELECT8_1_7]]
-; CHECK-NEXT:    [[TMP108:%.*]] = or i1 [[CMP12_2_7]], [[TMP105]]
-; CHECK-NEXT:    [[SPEC_SELECT8_2_7:%.*]] = select i1 [[CMP12_2_7]], i32 [[TMP107]], i32 [[SPEC_SELECT8_1_7]]
-; CHECK-NEXT:    [[TMP110:%.*]] = extractelement <32 x i32> [[TMP8]], i32 31
-; CHECK-NEXT:    [[CMP12_3_7:%.*]] = icmp slt i32 [[TMP110]], [[SPEC_SELECT8_2_7]]
-; CHECK-NEXT:    [[TMP111:%.*]] = or i1 [[CMP12_3_7]], [[TMP108]]
+; CHECK-NEXT:    [[TMP87:%.*]] = extractelement <4 x i32> [[TMP77]], i32 3
+; CHECK-NEXT:    [[SPEC_SELECT8_3_6:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP87]], i32 [[SPEC_SELECT8_2_6]])
+; CHECK-NEXT:    [[TMP88:%.*]] = sub <4 x i32> [[TMP11]], [[TMP7]]
+; CHECK-NEXT:    [[TMP101:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> [[TMP88]], i1 true)
+; CHECK-NEXT:    [[TMP90:%.*]] = extractelement <4 x i32> [[TMP101]], i32 0
+; CHECK-NEXT:    [[SPEC_SELECT8_7:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP90]], i32 [[SPEC_SELECT8_3_6]])
+; CHECK-NEXT:    [[TMP91:%.*]] = extractelement <4 x i32> [[TMP101]], i32 1
+; CHECK-NEXT:    [[SPEC_SELECT8_1_7:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP91]], i32 [[SPEC_SELECT8_7]])
+; CHECK-NEXT:    [[TMP92:%.*]] = extractelement <4 x i32> [[TMP101]], i32 2
+; CHECK-NEXT:    [[SPEC_SELECT8_2_7:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP92]], i32 [[SPEC_SELECT8_1_7]])
+; CHECK-NEXT:    [[TMP93:%.*]] = insertelement <4 x i32> poison, i32 [[SPEC_SELECT8_3_6]], i32 0
+; CHECK-NEXT:    [[TMP94:%.*]] = insertelement <4 x i32> [[TMP93]], i32 [[SPEC_SELECT8_7]], i32 1
+; CHECK-NEXT:    [[TMP95:%.*]] = insertelement <4 x i32> [[TMP94]], i32 [[SPEC_SELECT8_1_7]], i32 2
+; CHECK-NEXT:    [[TMP96:%.*]] = insertelement <4 x i32> [[TMP95]], i32 [[SPEC_SELECT8_2_7]], i32 3
+; CHECK-NEXT:    [[TMP97:%.*]] = icmp slt <4 x i32> [[TMP101]], [[TMP96]]
+; CHECK-NEXT:    [[TMP111:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP97]])
 ; CHECK-NEXT:    [[SPEC_SELECT_3_7:%.*]] = select i1 [[TMP111]], i32 7, i32 [[SPEC_SELECT_3_6]]
-; CHECK-NEXT:    [[SPEC_SELECT8_3_7]] = select i1 [[CMP12_3_7]], i32 [[TMP110]], i32 [[SPEC_SELECT8_2_7]]
-; CHECK-NEXT:    [[K:%.*]] = getelementptr inbounds [366 x i32], ptr @l, i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP99:%.*]] = extractelement <4 x i32> [[TMP101]], i32 3
+; CHECK-NEXT:    [[SPEC_SELECT8_3_7]] = call i32 @llvm.smin.i32(i32 [[TMP99]], i32 [[SPEC_SELECT8_2_7]])
+; CHECK-NEXT:    [[K:%.*]] = getelementptr inbounds [4 x i8], ptr @l, i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store i32 [[SPEC_SELECT_3_7]], ptr [[K]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    br label [[FOR_COND]]
@@ -183,228 +193,159 @@ for.cond:                                         ; preds = %for.cond, %entry
   %32 = trunc i64 %indvars.iv to i32
   %33 = add i32 %32, -183
   %sub = sub i32 %33, %0
-  %34 = icmp slt i32 %sub, 0
-  %neg = sub nsw i32 0, %sub
-  %35 = select i1 %34, i32 %neg, i32 %sub
-  %cmp12 = icmp slt i32 %35, %b.0
-  %spec.select8 = select i1 %cmp12, i32 %35, i32 %b.0
+  %34 = call i32 @llvm.abs.i32(i32 %sub, i1 true)
+  %spec.select8 = call i32 @llvm.smin.i32(i32 %34, i32 %b.0)
   %sub.1 = sub i32 %33, %1
-  %36 = icmp slt i32 %sub.1, 0
-  %neg.1 = sub nsw i32 0, %sub.1
-  %37 = select i1 %36, i32 %neg.1, i32 %sub.1
-  %cmp12.1 = icmp slt i32 %37, %spec.select8
-  %spec.select8.1 = select i1 %cmp12.1, i32 %37, i32 %spec.select8
+  %35 = call i32 @llvm.abs.i32(i32 %sub.1, i1 true)
+  %spec.select8.1 = call i32 @llvm.smin.i32(i32 %35, i32 %spec.select8)
   %sub.2 = sub i32 %33, %2
-  %38 = icmp slt i32 %sub.2, 0
-  %neg.2 = sub nsw i32 0, %sub.2
-  %39 = select i1 %38, i32 %neg.2, i32 %sub.2
-  %cmp12.2 = icmp slt i32 %39, %spec.select8.1
-  %spec.select8.2 = select i1 %cmp12.2, i32 %39, i32 %spec.select8.1
+  %36 = call i32 @llvm.abs.i32(i32 %sub.2, i1 true)
+  %spec.select8.2 = call i32 @llvm.smin.i32(i32 %36, i32 %spec.select8.1)
   %sub.3 = sub i32 %33, %3
-  %40 = icmp slt i32 %sub.3, 0
-  %neg.3 = sub nsw i32 0, %sub.3
-  %41 = select i1 %40, i32 %neg.3, i32 %sub.3
-  %cmp12.3 = icmp slt i32 %41, %spec.select8.2
-  %spec.select8.3 = select i1 %cmp12.3, i32 %41, i32 %spec.select8.2
+  %37 = call i32 @llvm.abs.i32(i32 %sub.3, i1 true)
+  %spec.select8.3 = call i32 @llvm.smin.i32(i32 %37, i32 %spec.select8.2)
   %sub.116 = sub i32 %33, %4
-  %42 = icmp slt i32 %sub.116, 0
-  %neg.117 = sub nsw i32 0, %sub.116
-  %43 = select i1 %42, i32 %neg.117, i32 %sub.116
-  %cmp12.118 = icmp slt i32 %43, %spec.select8.3
-  %spec.select8.120 = select i1 %cmp12.118, i32 %43, i32 %spec.select8.3
+  %38 = call i32 @llvm.abs.i32(i32 %sub.116, i1 true)
+  %cmp12.118 = icmp slt i32 %38, %spec.select8.3
+  %spec.select8.120 = call i32 @llvm.smin.i32(i32 %38, i32 %spec.select8.3)
   %sub.1.1 = sub i32 %33, %5
-  %44 = icmp slt i32 %sub.1.1, 0
-  %neg.1.1 = sub nsw i32 0, %sub.1.1
-  %45 = select i1 %44, i32 %neg.1.1, i32 %sub.1.1
-  %cmp12.1.1 = icmp slt i32 %45, %spec.select8.120
+  %39 = call i32 @llvm.abs.i32(i32 %sub.1.1, i1 true)
+  %cmp12.1.1 = icmp slt i32 %39, %spec.select8.120
   %narrow = or i1 %cmp12.1.1, %cmp12.118
-  %spec.select8.1.1 = select i1 %cmp12.1.1, i32 %45, i32 %spec.select8.120
+  %spec.select8.1.1 = call i32 @llvm.smin.i32(i32 %39, i32 %spec.select8.120)
   %sub.2.1 = sub i32 %33, %6
-  %46 = icmp slt i32 %sub.2.1, 0
-  %neg.2.1 = sub nsw i32 0, %sub.2.1
-  %47 = select i1 %46, i32 %neg.2.1, i32 %sub.2.1
-  %cmp12.2.1 = icmp slt i32 %47, %spec.select8.1.1
+  %40 = call i32 @llvm.abs.i32(i32 %sub.2.1, i1 true)
+  %cmp12.2.1 = icmp slt i32 %40, %spec.select8.1.1
   %narrow34 = or i1 %cmp12.2.1, %narrow
-  %spec.select8.2.1 = select i1 %cmp12.2.1, i32 %47, i32 %spec.select8.1.1
+  %spec.select8.2.1 = call i32 @llvm.smin.i32(i32 %40, i32 %spec.select8.1.1)
   %sub.3.1 = sub i32 %33, %7
-  %48 = icmp slt i32 %sub.3.1, 0
-  %neg.3.1 = sub nsw i32 0, %sub.3.1
-  %49 = select i1 %48, i32 %neg.3.1, i32 %sub.3.1
-  %cmp12.3.1 = icmp slt i32 %49, %spec.select8.2.1
+  %41 = call i32 @llvm.abs.i32(i32 %sub.3.1, i1 true)
+  %cmp12.3.1 = icmp slt i32 %41, %spec.select8.2.1
   %narrow35 = or i1 %cmp12.3.1, %narrow34
   %spec.select.3.1 = zext i1 %narrow35 to i32
-  %spec.select8.3.1 = select i1 %cmp12.3.1, i32 %49, i32 %spec.select8.2.1
+  %spec.select8.3.1 = call i32 @llvm.smin.i32(i32 %41, i32 %spec.select8.2.1)
   %sub.222 = sub i32 %33, %8
-  %50 = icmp slt i32 %sub.222, 0
-  %neg.223 = sub nsw i32 0, %sub.222
-  %51 = select i1 %50, i32 %neg.223, i32 %sub.222
-  %cmp12.224 = icmp slt i32 %51, %spec.select8.3.1
-  %spec.select8.226 = select i1 %cmp12.224, i32 %51, i32 %spec.select8.3.1
+  %42 = call i32 @llvm.abs.i32(i32 %sub.222, i1 true)
+  %cmp12.224 = icmp slt i32 %42, %spec.select8.3.1
+  %spec.select8.226 = call i32 @llvm.smin.i32(i32 %42, i32 %spec.select8.3.1)
   %sub.1.2 = sub i32 %33, %9
-  %52 = icmp slt i32 %sub.1.2, 0
-  %neg.1.2 = sub nsw i32 0, %sub.1.2
-  %53 = select i1 %52, i32 %neg.1.2, i32 %sub.1.2
-  %cmp12.1.2 = icmp slt i32 %53, %spec.select8.226
-  %54 = or i1 %cmp12.1.2, %cmp12.224
-  %spec.select8.1.2 = select i1 %cmp12.1.2, i32 %53, i32 %spec.select8.226
+  %43 = call i32 @llvm.abs.i32(i32 %sub.1.2, i1 true)
+  %cmp12.1.2 = icmp slt i32 %43, %spec.select8.226
+  %44 = or i1 %cmp12.1.2, %cmp12.224
+  %spec.select8.1.2 = call i32 @llvm.smin.i32(i32 %43, i32 %spec.select8.226)
   %sub.2.2 = sub i32 %33, %10
-  %55 = icmp slt i32 %sub.2.2, 0
-  %neg.2.2 = sub nsw i32 0, %sub.2.2
-  %56 = select i1 %55, i32 %neg.2.2, i32 %sub.2.2
-  %cmp12.2.2 = icmp slt i32 %56, %spec.select8.1.2
-  %57 = or i1 %cmp12.2.2, %54
-  %spec.select8.2.2 = select i1 %cmp12.2.2, i32 %56, i32 %spec.select8.1.2
+  %45 = call i32 @llvm.abs.i32(i32 %sub.2.2, i1 true)
+  %cmp12.2.2 = icmp slt i32 %45, %spec.select8.1.2
+  %46 = or i1 %cmp12.2.2, %44
+  %spec.select8.2.2 = call i32 @llvm.smin.i32(i32 %45, i32 %spec.select8.1.2)
   %sub.3.2 = sub i32 %33, %11
-  %58 = icmp slt i32 %sub.3.2, 0
-  %neg.3.2 = sub nsw i32 0, %sub.3.2
-  %59 = select i1 %58, i32 %neg.3.2, i32 %sub.3.2
-  %cmp12.3.2 = icmp slt i32 %59, %spec.select8.2.2
-  %60 = or i1 %cmp12.3.2, %57
-  %spec.select.3.2 = select i1 %60, i32 2, i32 %spec.select.3.1
-  %spec.select8.3.2 = select i1 %cmp12.3.2, i32 %59, i32 %spec.select8.2.2
+  %47 = call i32 @llvm.abs.i32(i32 %sub.3.2, i1 true)
+  %cmp12.3.2 = icmp slt i32 %47, %spec.select8.2.2
+  %48 = or i1 %cmp12.3.2, %46
+  %spec.select.3.2 = select i1 %48, i32 2, i32 %spec.select.3.1
+  %spec.select8.3.2 = call i32 @llvm.smin.i32(i32 %47, i32 %spec.select8.2.2)
   %sub.328 = sub i32 %33, %12
-  %61 = icmp slt i32 %sub.328, 0
-  %neg.329 = sub nsw i32 0, %sub.328
-  %62 = select i1 %61, i32 %neg.329, i32 %sub.328
-  %cmp12.330 = icmp slt i32 %62, %spec.select8.3.2
-  %spec.select8.332 = select i1 %cmp12.330, i32 %62, i32 %spec.select8.3.2
+  %49 = call i32 @llvm.abs.i32(i32 %sub.328, i1 true)
+  %cmp12.330 = icmp slt i32 %49, %spec.select8.3.2
+  %spec.select8.332 = call i32 @llvm.smin.i32(i32 %49, i32 %spec.select8.3.2)
   %sub.1.3 = sub i32 %33, %13
-  %63 = icmp slt i32 %sub.1.3, 0
-  %neg.1.3 = sub nsw i32 0, %sub.1.3
-  %64 = select i1 %63, i32 %neg.1.3, i32 %sub.1.3
-  %cmp12.1.3 = icmp slt i32 %64, %spec.select8.332
-  %65 = or i1 %cmp12.1.3, %cmp12.330
-  %spec.select8.1.3 = select i1 %cmp12.1.3, i32 %64, i32 %spec.select8.332
+  %50 = call i32 @llvm.abs.i32(i32 %sub.1.3, i1 true)
+  %cmp12.1.3 = icmp slt i32 %50, %spec.select8.332
+  %51 = or i1 %cmp12.1.3, %cmp12.330
+  %spec.select8.1.3 = call i32 @llvm.smin.i32(i32 %50, i32 %spec.select8.332)
   %sub.2.3 = sub i32 %33, %14
-  %66 = icmp slt i32 %sub.2.3, 0
-  %neg.2.3 = sub nsw i32 0, %sub.2.3
-  %67 = select i1 %66, i32 %neg.2.3, i32 %sub.2.3
-  %cmp12.2.3 = icmp slt i32 %67, %spec.select8.1.3
-  %68 = or i1 %cmp12.2.3, %65
-  %spec.select8.2.3 = select i1 %cmp12.2.3, i32 %67, i32 %spec.select8.1.3
+  %52 = call i32 @llvm.abs.i32(i32 %sub.2.3, i1 true)
+  %cmp12.2.3 = icmp slt i32 %52, %spec.select8.1.3
+  %53 = or i1 %cmp12.2.3, %51
+  %spec.select8.2.3 = call i32 @llvm.smin.i32(i32 %52, i32 %spec.select8.1.3)
   %sub.3.3 = sub i32 %33, %15
-  %69 = icmp slt i32 %sub.3.3, 0
-  %neg.3.3 = sub nsw i32 0, %sub.3.3
-  %70 = select i1 %69, i32 %neg.3.3, i32 %sub.3.3
-  %cmp12.3.3 = icmp slt i32 %70, %spec.select8.2.3
-  %71 = or i1 %cmp12.3.3, %68
-  %spec.select.3.3 = select i1 %71, i32 3, i32 %spec.select.3.2
-  %spec.select8.3.3 = select i1 %cmp12.3.3, i32 %70, i32 %spec.select8.2.3
+  %54 = call i32 @llvm.abs.i32(i32 %sub.3.3, i1 true)
+  %cmp12.3.3 = icmp slt i32 %54, %spec.select8.2.3
+  %55 = or i1 %cmp12.3.3, %53
+  %spec.select.3.3 = select i1 %55, i32 3, i32 %spec.select.3.2
+  %spec.select8.3.3 = call i32 @llvm.smin.i32(i32 %54, i32 %spec.select8.2.3)
   %sub.4 = sub i32 %33, %16
-  %72 = icmp slt i32 %sub.4, 0
-  %neg.4 = sub nsw i32 0, %sub.4
-  %73 = select i1 %72, i32 %neg.4, i32 %sub.4
-  %cmp12.4 = icmp slt i32 %73, %spec.select8.3.3
-  %spec.select8.4 = select i1 %cmp12.4, i32 %73, i32 %spec.select8.3.3
+  %56 = call i32 @llvm.abs.i32(i32 %sub.4, i1 true)
+  %cmp12.4 = icmp slt i32 %56, %spec.select8.3.3
+  %spec.select8.4 = call i32 @llvm.smin.i32(i32 %56, i32 %spec.select8.3.3)
   %sub.1.4 = sub i32 %33, %17
-  %74 = icmp slt i32 %sub.1.4, 0
-  %neg.1.4 = sub nsw i32 0, %sub.1.4
-  %75 = select i1 %74, i32 %neg.1.4, i32 %sub.1.4
-  %cmp12.1.4 = icmp slt i32 %75, %spec.select8.4
-  %76 = or i1 %cmp12.1.4, %cmp12.4
-  %spec.select8.1.4 = select i1 %cmp12.1.4, i32 %75, i32 %spec.select8.4
+  %57 = call i32 @llvm.abs.i32(i32 %sub.1.4, i1 true)
+  %cmp12.1.4 = icmp slt i32 %57, %spec.select8.4
+  %58 = or i1 %cmp12.1.4, %cmp12.4
+  %spec.select8.1.4 = call i32 @llvm.smin.i32(i32 %57, i32 %spec.select8.4)
   %sub.2.4 = sub i32 %33, %18
-  %77 = icmp slt i32 %sub.2.4, 0
-  %neg.2.4 = sub nsw i32 0, %sub.2.4
-  %78 = select i1 %77, i32 %neg.2.4, i32 %sub.2.4
-  %cmp12.2.4 = icmp slt i32 %78, %spec.select8.1.4
-  %79 = or i1 %cmp12.2.4, %76
-  %spec.select8.2.4 = select i1 %cmp12.2.4, i32 %78, i32 %spec.select8.1.4
+  %59 = call i32 @llvm.abs.i32(i32 %sub.2.4, i1 true)
+  %cmp12.2.4 = icmp slt i32 %59, %spec.select8.1.4
+  %60 = or i1 %cmp12.2.4, %58
+  %spec.select8.2.4 = call i32 @llvm.smin.i32(i32 %59, i32 %spec.select8.1.4)
   %sub.3.4 = sub i32 %33, %19
-  %80 = icmp slt i32 %sub.3.4, 0
-  %neg.3.4 = sub nsw i32 0, %sub.3.4
-  %81 = select i1 %80, i32 %neg.3.4, i32 %sub.3.4
-  %cmp12.3.4 = icmp slt i32 %81, %spec.select8.2.4
-  %82 = or i1 %cmp12.3.4, %79
-  %spec.select.3.4 = select i1 %82, i32 4, i32 %spec.select.3.3
-  %spec.select8.3.4 = select i1 %cmp12.3.4, i32 %81, i32 %spec.select8.2.4
+  %61 = call i32 @llvm.abs.i32(i32 %sub.3.4, i1 true)
+  %cmp12.3.4 = icmp slt i32 %61, %spec.select8.2.4
+  %62 = or i1 %cmp12.3.4, %60
+  %spec.select.3.4 = select i1 %62, i32 4, i32 %spec.select.3.3
+  %spec.select8.3.4 = call i32 @llvm.smin.i32(i32 %61, i32 %spec.select8.2.4)
   %sub.5 = sub i32 %33, %20
-  %83 = icmp slt i32 %sub.5, 0
-  %neg.5 = sub nsw i32 0, %sub.5
-  %84 = select i1 %83, i32 %neg.5, i32 %sub.5
-  %cmp12.5 = icmp slt i32 %84, %spec.select8.3.4
-  %spec.select8.5 = select i1 %cmp12.5, i32 %84, i32 %spec.select8.3.4
+  %63 = call i32 @llvm.abs.i32(i32 %sub.5, i1 true)
+  %cmp12.5 = icmp slt i32 %63, %spec.select8.3.4
+  %spec.select8.5 = call i32 @llvm.smin.i32(i32 %63, i32 %spec.select8.3.4)
   %sub.1.5 = sub i32 %33, %21
-  %85 = icmp slt i32 %sub.1.5, 0
-  %neg.1.5 = sub nsw i32 0, %sub.1.5
-  %86 = select i1 %85, i32 %neg.1.5, i32 %sub.1.5
-  %cmp12.1.5 = icmp slt i32 %86, %spec.select8.5
-  %87 = or i1 %cmp12.1.5, %cmp12.5
-  %spec.select8.1.5 = select i1 %cmp12.1.5, i32 %86, i32 %spec.select8.5
+  %64 = call i32 @llvm.abs.i32(i32 %sub.1.5, i1 true)
+  %cmp12.1.5 = icmp slt i32 %64, %spec.select8.5
+  %65 = or i1 %cmp12.1.5, %cmp12.5
+  %spec.select8.1.5 = call i32 @llvm.smin.i32(i32 %64, i32 %spec.select8.5)
   %sub.2.5 = sub i32 %33, %22
-  %88 = icmp slt i32 %sub.2.5, 0
-  %neg.2.5 = sub nsw i32 0, %sub.2.5
-  %89 = select i1 %88, i32 %neg.2.5, i32 %sub.2.5
-  %cmp12.2.5 = icmp slt i32 %89, %spec.select8.1.5
-  %90 = or i1 %cmp12.2.5, %87
-  %spec.select8.2.5 = select i1 %cmp12.2.5, i32 %89, i32 %spec.select8.1.5
+  %66 = call i32 @llvm.abs.i32(i32 %sub.2.5, i1 true)
+  %cmp12.2.5 = icmp slt i32 %66, %spec.select8.1.5
+  %67 = or i1 %cmp12.2.5, %65
+  %spec.select8.2.5 = call i32 @llvm.smin.i32(i32 %66, i32 %spec.select8.1.5)
   %sub.3.5 = sub i32 %33, %23
-  %91 = icmp slt i32 %sub.3.5, 0
-  %neg.3.5 = sub nsw i32 0, %sub.3.5
-  %92 = select i1 %91, i32 %neg.3.5, i32 %sub.3.5
-  %cmp12.3.5 = icmp slt i32 %92, %spec.select8.2.5
-  %93 = or i1 %cmp12.3.5, %90
-  %spec.select.3.5 = select i1 %93, i32 5, i32 %spec.select.3.4
-  %spec.select8.3.5 = select i1 %cmp12.3.5, i32 %92, i32 %spec.select8.2.5
+  %68 = call i32 @llvm.abs.i32(i32 %sub.3.5, i1 true)
+  %cmp12.3.5 = icmp slt i32 %68, %spec.select8.2.5
+  %69 = or i1 %cmp12.3.5, %67
+  %spec.select.3.5 = select i1 %69, i32 5, i32 %spec.select.3.4
+  %spec.select8.3.5 = call i32 @llvm.smin.i32(i32 %68, i32 %spec.select8.2.5)
   %sub.6 = sub i32 %33, %24
-  %94 = icmp slt i32 %sub.6, 0
-  %neg.6 = sub nsw i32 0, %sub.6
-  %95 = select i1 %94, i32 %neg.6, i32 %sub.6
-  %cmp12.6 = icmp slt i32 %95, %spec.select8.3.5
-  %spec.select8.6 = select i1 %cmp12.6, i32 %95, i32 %spec.select8.3.5
+  %70 = call i32 @llvm.abs.i32(i32 %sub.6, i1 true)
+  %cmp12.6 = icmp slt i32 %70, %spec.select8.3.5
+  %spec.select8.6 = call i32 @llvm.smin.i32(i32 %70, i32 %spec.select8.3.5)
   %sub.1.6 = sub i32 %33, %25
-  %96 = icmp slt i32 %sub.1.6, 0
-  %neg.1.6 = sub nsw i32 0, %sub.1.6
-  %97 = select i1 %96, i32 %neg.1.6, i32 %sub.1.6
-  %cmp12.1.6 = icmp slt i32 %97, %spec.select8.6
-  %98 = or i1 %cmp12.1.6, %cmp12.6
-  %spec.select8.1.6 = select i1 %cmp12.1.6, i32 %97, i32 %spec.select8.6
+  %71 = call i32 @llvm.abs.i32(i32 %sub.1.6, i1 true)
+  %cmp12.1.6 = icmp slt i32 %71, %spec.select8.6
+  %72 = or i1 %cmp12.1.6, %cmp12.6
+  %spec.select8.1.6 = call i32 @llvm.smin.i32(i32 %71, i32 %spec.select8.6)
   %sub.2.6 = sub i32 %33, %26
-  %99 = icmp slt i32 %sub.2.6, 0
-  %neg.2.6 = sub nsw i32 0, %sub.2.6
-  %100 = select i1 %99, i32 %neg.2.6, i32 %sub.2.6
-  %cmp12.2.6 = icmp slt i32 %100, %spec.select8.1.6
-  %101 = or i1 %cmp12.2.6, %98
-  %spec.select8.2.6 = select i1 %cmp12.2.6, i32 %100, i32 %spec.select8.1.6
+  %73 = call i32 @llvm.abs.i32(i32 %sub.2.6, i1 true)
+  %cmp12.2.6 = icmp slt i32 %73, %spec.select8.1.6
+  %74 = or i1 %cmp12.2.6, %72
+  %spec.select8.2.6 = call i32 @llvm.smin.i32(i32 %73, i32 %spec.select8.1.6)
   %sub.3.6 = sub i32 %33, %27
-  %102 = icmp slt i32 %sub.3.6, 0
-  %neg.3.6 = sub nsw i32 0, %sub.3.6
-  %103 = select i1 %102, i32 %neg.3.6, i32 %sub.3.6
-  %cmp12.3.6 = icmp slt i32 %103, %spec.select8.2.6
-  %104 = or i1 %cmp12.3.6, %101
-  %spec.select.3.6 = select i1 %104, i32 6, i32 %spec.select.3.5
-  %spec.select8.3.6 = select i1 %cmp12.3.6, i32 %103, i32 %spec.select8.2.6
+  %75 = call i32 @llvm.abs.i32(i32 %sub.3.6, i1 true)
+  %cmp12.3.6 = icmp slt i32 %75, %spec.select8.2.6
+  %76 = or i1 %cmp12.3.6, %74
+  %spec.select.3.6 = select i1 %76, i32 6, i32 %spec.select.3.5
+  %spec.select8.3.6 = call i32 @llvm.smin.i32(i32 %75, i32 %spec.select8.2.6)
   %sub.7 = sub i32 %33, %28
-  %105 = icmp slt i32 %sub.7, 0
-  %neg.7 = sub nsw i32 0, %sub.7
-  %106 = select i1 %105, i32 %neg.7, i32 %sub.7
-  %cmp12.7 = icmp slt i32 %106, %spec.select8.3.6
-  %spec.select8.7 = select i1 %cmp12.7, i32 %106, i32 %spec.select8.3.6
+  %77 = call i32 @llvm.abs.i32(i32 %sub.7, i1 true)
+  %cmp12.7 = icmp slt i32 %77, %spec.select8.3.6
+  %spec.select8.7 = call i32 @llvm.smin.i32(i32 %77, i32 %spec.select8.3.6)
   %sub.1.7 = sub i32 %33, %29
-  %107 = icmp slt i32 %sub.1.7, 0
-  %neg.1.7 = sub nsw i32 0, %sub.1.7
-  %108 = select i1 %107, i32 %neg.1.7, i32 %sub.1.7
-  %cmp12.1.7 = icmp slt i32 %108, %spec.select8.7
-  %109 = or i1 %cmp12.1.7, %cmp12.7
-  %spec.select8.1.7 = select i1 %cmp12.1.7, i32 %108, i32 %spec.select8.7
+  %78 = call i32 @llvm.abs.i32(i32 %sub.1.7, i1 true)
+  %cmp12.1.7 = icmp slt i32 %78, %spec.select8.7
+  %79 = or i1 %cmp12.1.7, %cmp12.7
+  %spec.select8.1.7 = call i32 @llvm.smin.i32(i32 %78, i32 %spec.select8.7)
   %sub.2.7 = sub i32 %33, %30
-  %110 = icmp slt i32 %sub.2.7, 0
-  %neg.2.7 = sub nsw i32 0, %sub.2.7
-  %111 = select i1 %110, i32 %neg.2.7, i32 %sub.2.7
-  %cmp12.2.7 = icmp slt i32 %111, %spec.select8.1.7
-  %112 = or i1 %cmp12.2.7, %109
-  %spec.select8.2.7 = select i1 %cmp12.2.7, i32 %111, i32 %spec.select8.1.7
+  %80 = call i32 @llvm.abs.i32(i32 %sub.2.7, i1 true)
+  %cmp12.2.7 = icmp slt i32 %80, %spec.select8.1.7
+  %81 = or i1 %cmp12.2.7, %79
+  %spec.select8.2.7 = call i32 @llvm.smin.i32(i32 %80, i32 %spec.select8.1.7)
   %sub.3.7 = sub i32 %33, %31
-  %113 = icmp slt i32 %sub.3.7, 0
-  %neg.3.7 = sub nsw i32 0, %sub.3.7
-  %114 = select i1 %113, i32 %neg.3.7, i32 %sub.3.7
-  %cmp12.3.7 = icmp slt i32 %114, %spec.select8.2.7
-  %115 = or i1 %cmp12.3.7, %112
-  %spec.select.3.7 = select i1 %115, i32 7, i32 %spec.select.3.6
-  %spec.select8.3.7 = select i1 %cmp12.3.7, i32 %114, i32 %spec.select8.2.7
-  %k = getelementptr inbounds [366 x i32], ptr @l, i64 0, i64 %indvars.iv
+  %82 = call i32 @llvm.abs.i32(i32 %sub.3.7, i1 true)
+  %cmp12.3.7 = icmp slt i32 %82, %spec.select8.2.7
+  %83 = or i1 %cmp12.3.7, %81
+  %spec.select.3.7 = select i1 %83, i32 7, i32 %spec.select.3.6
+  %spec.select8.3.7 = call i32 @llvm.smin.i32(i32 %82, i32 %spec.select8.2.7)
+  %k = getelementptr inbounds [4 x i8], ptr @l, i64 %indvars.iv
   store i32 %spec.select.3.7, ptr %k, align 4
   %indvars.iv.next = add i64 %indvars.iv, 1
   br label %for.cond
 }
-
diff --git a/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll b/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
index d1b7dcab99612..aa5e7b10a089b 100644
--- a/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
+++ b/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
@@ -31,13 +31,10 @@ define ptr @store_only_fp_tail() {
 ; NO-CANON-NEXT:    ret ptr null
 ;
 ; CANON-LABEL: define ptr @store_only_fp_tail() {
-; CANON-NEXT:    [[DOTSROA_4:%.*]] = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
-; CANON-NEXT:    [[DOTSROA_2:%.*]] = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
-; CANON-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DOTSROA_4]], ptr align 8 [[DOTSROA_2]], i64 44, i1 false)
 ; CANON-NEXT:    [[DST_SROA_0_0_VEC_INSERT:%.*]] = insertelement <4 x float> {{.*}}, float 0.000000e+00, i32 0
 ; CANON-NEXT:    store <4 x float> [[DST_SROA_0_0_VEC_INSERT]], ptr null, align 1
 ; CANON-NEXT:    store float 0.000000e+00, ptr getelementptr inbounds (i8, ptr null, i64 16), align 1
-; CANON-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 getelementptr inbounds (i8, ptr null, i64 20), ptr align 8 [[DOTSROA_4]], i64 44, i1 false)
+; CANON-NEXT:    store <11 x float> {{.*}}, ptr getelementptr inbounds (i8, ptr null, i64 20), align 1
 ; CANON-NEXT:    ret ptr null
 ;
   %dst = alloca %class.aiMatrix4x4t, align 4
diff --git a/llvm/test/Transforms/SROA/struct-to-vector.ll b/llvm/test/Transforms/SROA/struct-to-vector.ll
index ad7d3df129d6c..b7fb3cc97d66a 100644
--- a/llvm/test/Transforms/SROA/struct-to-vector.ll
+++ b/llvm/test/Transforms/SROA/struct-to-vector.ll
@@ -387,3 +387,34 @@ cond.end.ptr:
   call void @llvm.lifetime.end.p0(ptr %temp)
   ret void
 }
+
+%struct.i5x2 = type { i5, i5 }
+define void @struct_i5x2_memcpy_into_alloca(ptr %c) {
+; CHECK-LABEL: define void @struct_i5x2_memcpy_into_alloca(
+; CHECK-SAME: ptr [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[E:%.*]] = alloca [[STRUCT_I5X2:%.*]], align 1
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[E]], ptr align 1 [[C]], i32 2, i1 true)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %e = alloca %struct.i5x2, align 1
+  call void @llvm.memcpy.p0.p0.i32(ptr align 1 %e, ptr align 1 %c, i32 2, i1 true)
+  ret void
+}
+
+%struct.i32x3 = type { i32, i32, i32 }
+define void @struct_i32x3_memcpy_into_alloca(ptr %c) {
+; CHECK-LABEL: define void @struct_i32x3_memcpy_into_alloca(
+; CHECK-SAME: ptr [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[E_SROA_0:%.*]] = alloca <3 x i32>, align 16
+; CHECK-NEXT:    [[E_SROA_0_0_COPYLOAD:%.*]] = load volatile <3 x i32>, ptr [[C]], align 4
+; CHECK-NEXT:    store volatile <3 x i32> [[E_SROA_0_0_COPYLOAD]], ptr [[E_SROA_0]], align 16
+; CHECK-NEXT:    ret void
+;
+entry:
+  %e = alloca %struct.i32x3, align 4
+  call void @llvm.memcpy.p0.p0.i32(ptr align 4 %e, ptr align 4 %c, i32 12, i1 true)
+  ret void
+}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/vcp-noipa.ll b/llvm/test/Transforms/WholeProgramDevirt/vcp-noipa.ll
new file mode 100644
index 0000000000000..7d1bdd2489028
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/vcp-noipa.ll
@@ -0,0 +1,30 @@
+; RUN: opt -S -passes=wholeprogramdevirt -whole-program-visibility %s | FileCheck %s
+
+; The `noipa` attribute prevents interprocedural function body inspection,
+; including this virtual constant propagation transformation.
+
+@vt1 = constant [1 x ptr] [ptr @vf1], !type !0
+@vt2 = constant [1 x ptr] [ptr @vf2], !type !0
+
+define i32 @vf1(ptr %this, i32 %arg) readnone noipa {
+  ret i32 %arg
+}
+
+define i32 @vf2(ptr %this, i32 %arg) readnone {
+  ret i32 %arg
+}
+
+; CHECK: define i32 @fn
+define i32 @fn(ptr %obj) {
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptr = load ptr, ptr %vtable
+  ; CHECK: call i32 %
+  %result = call i32 %fptr(ptr %obj, i32 1)
+  ret i32 %result
+}
+declare i1 @llvm.type.test(ptr, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Verifier/anyptr-bad.ll b/llvm/test/Verifier/anyptr-bad.ll
new file mode 100644
index 0000000000000..7f942b39efe48
--- /dev/null
+++ b/llvm/test/Verifier/anyptr-bad.ll
@@ -0,0 +1,11 @@
+; RUN: not opt -passes=verify < %s 2>&1 | FileCheck %s
+
+; CHECK: intrinsic return type (overload type 0) expected any pointer type, but got <2 x ptr>
+; CHECK-NEXT: ptr @llvm.returnaddress
+
+declare <2 x ptr> @llvm.returnaddress(i32)
+
+define void @foo() {
+  call <2 x ptr> @llvm.returnaddress(i32 0)
+  ret void
+}
diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-neon-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-neon-instructions.s
index 9417ee208a1fe..b5aa217ed3144 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-neon-instructions.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-neon-instructions.s
@@ -1385,44 +1385,44 @@ zip2	v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      8     5.00                        fsqrt	v0.8h, v0.8h
 # CHECK-NEXT:  1      4     0.50                        fsub	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  1      3     0.50    *                   ld1	{ v0.16b }, [x0]
-# CHECK-NEXT:  2      4     1.00    *                   ld1	{ v0.2d, v1.2d, v2.2d }, [x0], #48
-# CHECK-NEXT:  1      4     1.00    *                   ld1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
-# CHECK-NEXT:  2      4     1.00    *                   ld1	{ v0.4s, v1.4s }, [sp], #32
-# CHECK-NEXT:  1      4     1.00    *                   ld1	{ v0.4s, v1.4s, v2.4s }, [sp]
-# CHECK-NEXT:  2      4     1.00    *                   ld1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
-# CHECK-NEXT:  2      4     1.00    *                   ld1	{ v0.8h }, [x15], x2
-# CHECK-NEXT:  1      3     0.50    *                   ld1	{ v0.8h, v1.8h }, [x15]
+# CHECK-NEXT:  2      4     2.00    *                   ld1	{ v0.2d, v1.2d, v2.2d }, [x0], #48
+# CHECK-NEXT:  1      4     2.00    *                   ld1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+# CHECK-NEXT:  2      3     1.00    *                   ld1	{ v0.4s, v1.4s }, [sp], #32
+# CHECK-NEXT:  1      4     2.00    *                   ld1	{ v0.4s, v1.4s, v2.4s }, [sp]
+# CHECK-NEXT:  2      4     2.00    *                   ld1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
+# CHECK-NEXT:  2      3     0.50    *                   ld1	{ v0.8h }, [x15], x2
+# CHECK-NEXT:  1      3     1.00    *                   ld1	{ v0.8h, v1.8h }, [x15]
 # CHECK-NEXT:  1      3     0.50    *                   ld1	{ v0.b }[9], [x0]
 # CHECK-NEXT:  2      3     0.50    *                   ld1	{ v0.b }[9], [x0], #1
 # CHECK-NEXT:  1      3     0.50    *                   ld1r	{ v0.16b }, [x0]
 # CHECK-NEXT:  2      3     0.50    *                   ld1r	{ v0.16b }, [x0], #1
 # CHECK-NEXT:  1      3     0.50    *                   ld1r	{ v0.8h }, [x15]
 # CHECK-NEXT:  2      3     0.50    *                   ld1r	{ v0.8h }, [x15], #2
-# CHECK-NEXT:  2      6     2.00    *                   ld2	{ v0.16b, v1.16b }, [x0], x1
+# CHECK-NEXT:  2      4     2.00    *                   ld2	{ v0.16b, v1.16b }, [x0], x1
 # CHECK-NEXT:  1      4     1.00    *                   ld2	{ v0.8b, v1.8b }, [x0]
-# CHECK-NEXT:  1      4     1.00    *                   ld2	{ v0.h, v1.h }[7], [x15]
-# CHECK-NEXT:  2      4     1.00    *                   ld2	{ v0.h, v1.h }[7], [x15], #4
-# CHECK-NEXT:  1      4     1.00    *                   ld2r	{ v0.2d, v1.2d }, [x0]
-# CHECK-NEXT:  2      4     1.00    *                   ld2r	{ v0.2d, v1.2d }, [x0], #16
-# CHECK-NEXT:  1      4     1.00    *                   ld2r	{ v0.4s, v1.4s }, [sp]
-# CHECK-NEXT:  2      4     1.00    *                   ld2r	{ v0.4s, v1.4s }, [sp], #8
-# CHECK-NEXT:  1      5     1.50    *                   ld3	{ v0.4h, v1.4h, v2.4h }, [x15]
-# CHECK-NEXT:  2      5     1.50    *                   ld3	{ v0.8h, v1.8h, v2.8h }, [x15], x2
-# CHECK-NEXT:  1      4     1.00    *                   ld3	{ v0.s, v1.s, v2.s }[3], [sp]
-# CHECK-NEXT:  2      4     1.00    *                   ld3	{ v0.s, v1.s, v2.s }[3], [sp], x3
-# CHECK-NEXT:  1      4     1.00    *                   ld3r	{ v0.4h, v1.4h, v2.4h }, [x15]
-# CHECK-NEXT:  2      4     1.00    *                   ld3r	{ v0.4h, v1.4h, v2.4h }, [x15], #6
-# CHECK-NEXT:  1      4     1.00    *                   ld3r	{ v0.8b, v1.8b, v2.8b }, [x0]
-# CHECK-NEXT:  2      4     1.00    *                   ld3r	{ v0.8b, v1.8b, v2.8b }, [x0], #3
-# CHECK-NEXT:  1      6     2.00    *                   ld4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
-# CHECK-NEXT:  2      6     2.00    *                   ld4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
-# CHECK-NEXT:  1      4     1.00    *                   ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0]
-# CHECK-NEXT:  2      4     1.00    *                   ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0], #32
-# CHECK-NEXT:  2      4     1.00    *                   ld4	{ v0.h, v1.h, v2.h, v3.h }[7], [x0], x0
-# CHECK-NEXT:  1      4     1.00    *                   ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp]
-# CHECK-NEXT:  2      4     1.00    *                   ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp], x7
-# CHECK-NEXT:  1      4     1.00    *                   ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
-# CHECK-NEXT:  2      4     1.00    *                   ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp], x30
+# CHECK-NEXT:  1      4     2.00    *                   ld2	{ v0.h, v1.h }[7], [x15]
+# CHECK-NEXT:  2      4     2.00    *                   ld2	{ v0.h, v1.h }[7], [x15], #4
+# CHECK-NEXT:  1      3     1.00    *                   ld2r	{ v0.2d, v1.2d }, [x0]
+# CHECK-NEXT:  2      3     1.00    *                   ld2r	{ v0.2d, v1.2d }, [x0], #16
+# CHECK-NEXT:  1      3     1.00    *                   ld2r	{ v0.4s, v1.4s }, [sp]
+# CHECK-NEXT:  2      3     1.00    *                   ld2r	{ v0.4s, v1.4s }, [sp], #8
+# CHECK-NEXT:  1      5     3.00    *                   ld3	{ v0.4h, v1.4h, v2.4h }, [x15]
+# CHECK-NEXT:  2      5     3.00    *                   ld3	{ v0.8h, v1.8h, v2.8h }, [x15], x2
+# CHECK-NEXT:  1      5     3.00    *                   ld3	{ v0.s, v1.s, v2.s }[3], [sp]
+# CHECK-NEXT:  2      5     3.00    *                   ld3	{ v0.s, v1.s, v2.s }[3], [sp], x3
+# CHECK-NEXT:  1      4     2.00    *                   ld3r	{ v0.4h, v1.4h, v2.4h }, [x15]
+# CHECK-NEXT:  2      4     2.00    *                   ld3r	{ v0.4h, v1.4h, v2.4h }, [x15], #6
+# CHECK-NEXT:  1      4     2.00    *                   ld3r	{ v0.8b, v1.8b, v2.8b }, [x0]
+# CHECK-NEXT:  2      4     2.00    *                   ld3r	{ v0.8b, v1.8b, v2.8b }, [x0], #3
+# CHECK-NEXT:  1      5     3.00    *                   ld4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+# CHECK-NEXT:  2      5     3.00    *                   ld4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
+# CHECK-NEXT:  1      6     4.00    *                   ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0]
+# CHECK-NEXT:  2      6     4.00    *                   ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0], #32
+# CHECK-NEXT:  2      6     4.00    *                   ld4	{ v0.h, v1.h, v2.h, v3.h }[7], [x0], x0
+# CHECK-NEXT:  1      4     2.00    *                   ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp]
+# CHECK-NEXT:  2      4     2.00    *                   ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp], x7
+# CHECK-NEXT:  1      4     2.00    *                   ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+# CHECK-NEXT:  2      4     2.00    *                   ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp], x30
 # CHECK-NEXT:  1      4     0.50                        mla	v0.8b, v0.8b, v0.8b
 # CHECK-NEXT:  1      4     0.50                        mls	v0.4h, v0.4h, v0.4h
 # CHECK-NEXT:  1      4     0.50                        mov	b0, v0.b[15]
@@ -2148,7 +2148,7 @@ zip2	v0.8h, v0.8h, v0.8h
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1.0]  [1.1]  [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10.0] [10.1] [11]
-# CHECK-NEXT:  -      -      -      -      -     39.00  91.00   -      -     501.00 501.00 3.00   3.00   197.00
+# CHECK-NEXT:  -      -      -      -      -     71.00  123.00  -      -     501.00 501.00 3.00   3.00   197.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1.0]  [1.1]  [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10.0] [10.1] [11]   Instructions:
@@ -2467,13 +2467,13 @@ zip2	v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     5.00   fsqrt	v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsub	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1	{ v0.16b }, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld1	{ v0.2d, v1.2d, v2.2d }, [x0], #48
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -      -     ld1	{ v0.2d, v1.2d, v2.2d }, [x0], #48
+# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -      -     ld1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
 # CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld1	{ v0.4s, v1.4s }, [sp], #32
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld1	{ v0.4s, v1.4s, v2.4s }, [sp]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld1	{ v0.8h }, [x15], x2
-# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1	{ v0.8h, v1.8h }, [x15]
+# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -      -     ld1	{ v0.4s, v1.4s, v2.4s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -      -     ld1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1	{ v0.8h }, [x15], x2
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld1	{ v0.8h, v1.8h }, [x15]
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1	{ v0.b }[9], [x0]
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1	{ v0.b }[9], [x0], #1
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1r	{ v0.16b }, [x0]
@@ -2482,29 +2482,29 @@ zip2	v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1r	{ v0.8h }, [x15], #2
 # CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -      -     ld2	{ v0.16b, v1.16b }, [x0], x1
 # CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld2	{ v0.8b, v1.8b }, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld2	{ v0.h, v1.h }[7], [x15]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld2	{ v0.h, v1.h }[7], [x15], #4
+# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -      -     ld2	{ v0.h, v1.h }[7], [x15]
+# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -      -     ld2	{ v0.h, v1.h }[7], [x15], #4
 # CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld2r	{ v0.2d, v1.2d }, [x0]
 # CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld2r	{ v0.2d, v1.2d }, [x0], #16
 # CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld2r	{ v0.4s, v1.4s }, [sp]
 # CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld2r	{ v0.4s, v1.4s }, [sp], #8
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -      -      -      -      -      -     ld3	{ v0.4h, v1.4h, v2.4h }, [x15]
-# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -      -      -      -      -      -     ld3	{ v0.8h, v1.8h, v2.8h }, [x15], x2
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld3	{ v0.s, v1.s, v2.s }[3], [sp]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld3	{ v0.s, v1.s, v2.s }[3], [sp], x3
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld3r	{ v0.4h, v1.4h, v2.4h }, [x15]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld3r	{ v0.4h, v1.4h, v2.4h }, [x15], #6
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld3r	{ v0.8b, v1.8b, v2.8b }, [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld3r	{ v0.8b, v1.8b, v2.8b }, [x0], #3
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -      -     ld4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
-# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -      -     ld4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0], #32
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld4	{ v0.h, v1.h, v2.h, v3.h }[7], [x0], x0
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp], x7
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
-# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp], x30
+# CHECK-NEXT:  -      -      -      -      -     3.00   3.00    -      -      -      -      -      -      -     ld3	{ v0.4h, v1.4h, v2.4h }, [x15]
+# CHECK-NEXT:  -      -      -      -      -     3.00   3.00    -      -      -      -      -      -      -     ld3	{ v0.8h, v1.8h, v2.8h }, [x15], x2
+# CHECK-NEXT:  -      -      -      -      -     3.00   3.00    -      -      -      -      -      -      -     ld3	{ v0.s, v1.s, v2.s }[3], [sp]
+# CHECK-NEXT:  -      -      -      -      -     3.00   3.00    -      -      -      -      -      -      -     ld3	{ v0.s, v1.s, v2.s }[3], [sp], x3
+# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -      -     ld3r	{ v0.4h, v1.4h, v2.4h }, [x15]
+# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -      -     ld3r	{ v0.4h, v1.4h, v2.4h }, [x15], #6
+# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -      -     ld3r	{ v0.8b, v1.8b, v2.8b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -      -     ld3r	{ v0.8b, v1.8b, v2.8b }, [x0], #3
+# CHECK-NEXT:  -      -      -      -      -     3.00   3.00    -      -      -      -      -      -      -     ld4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -     3.00   3.00    -      -      -      -      -      -      -     ld4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
+# CHECK-NEXT:  -      -      -      -      -     4.00   4.00    -      -      -      -      -      -      -     ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0]
+# CHECK-NEXT:  -      -      -      -      -     4.00   4.00    -      -      -      -      -      -      -     ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0], #32
+# CHECK-NEXT:  -      -      -      -      -     4.00   4.00    -      -      -      -      -      -      -     ld4	{ v0.h, v1.h, v2.h, v3.h }[7], [x0], x0
+# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -      -     ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp]
+# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -      -     ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp], x7
+# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -      -     ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -      -     ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp], x30
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mla	v0.8b, v0.8b, v0.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mls	v0.4h, v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	b0, v0.b[15]
diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-writeback.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-writeback.s
index fa87b28bfc02a..762240768cf9f 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-writeback.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-writeback.s
@@ -1162,28 +1162,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1701
+# CHECK-NEXT: Total Cycles:      1501
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.88
-# CHECK-NEXT: IPC:               0.59
+# CHECK-NEXT: uOps Per Cycle:    1.00
+# CHECK-NEXT: IPC:               0.67
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234567
+# CHECK-NEXT:                     012345
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeE .    .    . .   ld1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: [0,1]     . DE .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .  DeeeE  .    . .   ld1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: [0,3]     .    .DE  .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    . DeeE    . .   ld1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: [0,5]     .    .   DE    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    DeeE . .   ld1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: [0,7]     .    .    . DE . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .  DeeeE   ld1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: [0,9]     .    .    .    .DE   add	x0, x27, #1
+# CHECK:      [0,0]     DeeE .    .    .   ld1	{ v1.1d }, [x27], #8
+# CHECK-NEXT: [0,1]     . DE .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  DeeE   .    .   ld1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: [0,3]     .    DE   .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .DeeE.    .   ld1	{ v1.2s }, [x27], #8
+# CHECK-NEXT: [0,5]     .    .  DE.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .   DeeE  .   ld1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: [0,7]     .    .    .DE  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    . DeeE   ld1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: [0,9]     .    .    .   DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1208,28 +1208,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1801
+# CHECK-NEXT: Total Cycles:      1501
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.83
-# CHECK-NEXT: IPC:               0.56
+# CHECK-NEXT: uOps Per Cycle:    1.00
+# CHECK-NEXT: IPC:               0.67
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012345678
+# CHECK-NEXT:                     012345
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeE .    .    .  .   ld1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: [0,1]     . DE .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .  DeeeE  .    .  .   ld1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: [0,3]     .    .DE  .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    . DeeeE   .  .   ld1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: [0,5]     .    .    DE   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    .DeeE.  .   ld1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: [0,7]     .    .    .  DE.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .   DeeeE   ld1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .    . DE   add	x0, x27, #1
+# CHECK:      [0,0]     DeeE .    .    .   ld1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: [0,1]     . DE .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  DeeE   .    .   ld1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: [0,3]     .    DE   .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .DeeE.    .   ld1	{ v1.16b }, [x27], #16
+# CHECK-NEXT: [0,5]     .    .  DE.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .   DeeE  .   ld1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .    .DE  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    . DeeE   ld1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .   DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1254,28 +1254,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1701
+# CHECK-NEXT: Total Cycles:      1501
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.88
-# CHECK-NEXT: IPC:               0.59
+# CHECK-NEXT: uOps Per Cycle:    1.00
+# CHECK-NEXT: IPC:               0.67
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234567
+# CHECK-NEXT:                     012345
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeE .    .    . .   ld1	{ v1.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     . DE .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .  DeeE   .    . .   ld1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .    DE   .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .DeeeE    . .   ld1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     .    .   DE    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    DeeE . .   ld1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .    .    . DE . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .  DeeeE   ld1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .    .DE   add	x0, x27, #1
+# CHECK:      [0,0]     DeeE .    .    .   ld1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     . DE .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  DeeE   .    .   ld1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .    DE   .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .DeeE.    .   ld1	{ v1.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .    .  DE.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .   DeeE  .   ld1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .    .DE  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    . DeeE   ld1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .   DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1300,28 +1300,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2001
+# CHECK-NEXT: Total Cycles:      1501
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.75
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    1.00
+# CHECK-NEXT: IPC:               0.67
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0
+# CHECK-NEXT:                     012345
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeE.    .    .    .   ld1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     .  DE.    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .   DeeeE .    .    .   ld1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,3]     .    . DE .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .  DeeeE  .    .   ld1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,5]     .    .    .DE  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    . DeeeE   .   ld1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,7]     .    .    .    DE   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .DeeeE   ld1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,9]     .    .    .    .   DE   add	x0, x27, #1
+# CHECK:      [0,0]     DeeE .    .    .   ld1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     . DE .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  DeeE   .    .   ld1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,3]     .    DE   .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .DeeE.    .   ld1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,5]     .    .  DE.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .   DeeE  .   ld1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,7]     .    .    .DE  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    . DeeE   ld1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,9]     .    .    .   DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1346,28 +1346,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2001
+# CHECK-NEXT: Total Cycles:      1501
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.75
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    1.00
+# CHECK-NEXT: IPC:               0.67
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0
+# CHECK-NEXT:                     012345
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeE.    .    .    .   ld1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,1]     .  DE.    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .   DeeeE .    .    .   ld1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,3]     .    . DE .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .  DeeeE  .    .   ld1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,5]     .    .    .DE  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    . DeeeE   .   ld1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,7]     .    .    .    DE   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .DeeeE   ld1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .    .   DE   add	x0, x27, #1
+# CHECK:      [0,0]     DeeE .    .    .   ld1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,1]     . DE .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  DeeE   .    .   ld1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,3]     .    DE   .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .DeeE.    .   ld1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,5]     .    .  DE.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .   DeeE  .   ld1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,7]     .    .    .DE  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    . DeeE   ld1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .   DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1392,28 +1392,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2001
+# CHECK-NEXT: Total Cycles:      1501
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.75
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    1.00
+# CHECK-NEXT: IPC:               0.67
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0
+# CHECK-NEXT:                     012345
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeE.    .    .    .   ld1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     .  DE.    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .   DeeeE .    .    .   ld1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     .    . DE .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .  DeeeE  .    .   ld1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     .    .    .DE  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    . DeeeE   .   ld1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .    .    .    DE   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .DeeeE   ld1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .    .   DE   add	x0, x27, #1
+# CHECK:      [0,0]     DeeE .    .    .   ld1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     . DE .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  DeeE   .    .   ld1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .    DE   .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .DeeE.    .   ld1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     .    .  DE.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .   DeeE  .   ld1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .    .DE  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    . DeeE   ld1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .   DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1438,28 +1438,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2001
+# CHECK-NEXT: Total Cycles:      1801
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.75
-# CHECK-NEXT: IPC:               0.50
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: uOps Per Cycle:    0.83
+# CHECK-NEXT: IPC:               0.56
+# CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeE.    .    .    .   ld1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     .  DE.    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .   DeeeE .    .    .   ld1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     .    . DE .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .  DeeeE  .    .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,5]     .    .    .DE  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    . DeeeE   .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,7]     .    .    .    DE   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .DeeeE   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,9]     .    .    .    .   DE   add	x0, x27, #1
+# CHECK:      [0,0]     DeeE .    .    .  .   ld1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     . DE .    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  DeeE   .    .  .   ld1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .    DE   .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .DeeeE    .  .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,5]     .    .   DE    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    DeeeE.  .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,7]     .    .    .  DE.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .   DeeeE   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,9]     .    .    .    . DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1490,7 +1490,7 @@ add x0, x27, 1
 # CHECK:      Dispatch Width:    3
 # CHECK-NEXT: uOps Per Cycle:    0.75
 # CHECK-NEXT: IPC:               0.50
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
@@ -1536,7 +1536,7 @@ add x0, x27, 1
 # CHECK:      Dispatch Width:    3
 # CHECK-NEXT: uOps Per Cycle:    0.75
 # CHECK-NEXT: IPC:               0.50
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
@@ -1582,7 +1582,7 @@ add x0, x27, 1
 # CHECK:      Dispatch Width:    3
 # CHECK-NEXT: uOps Per Cycle:    0.75
 # CHECK-NEXT: IPC:               0.50
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
@@ -1628,7 +1628,7 @@ add x0, x27, 1
 # CHECK:      Dispatch Width:    3
 # CHECK-NEXT: uOps Per Cycle:    0.75
 # CHECK-NEXT: IPC:               0.50
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
@@ -1674,7 +1674,7 @@ add x0, x27, 1
 # CHECK:      Dispatch Width:    3
 # CHECK-NEXT: uOps Per Cycle:    0.75
 # CHECK-NEXT: IPC:               0.50
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
@@ -1720,7 +1720,7 @@ add x0, x27, 1
 # CHECK:      Dispatch Width:    3
 # CHECK-NEXT: uOps Per Cycle:    0.79
 # CHECK-NEXT: IPC:               0.53
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: Block RThroughput: 8.5
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
@@ -1990,28 +1990,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2001
+# CHECK-NEXT: Total Cycles:      1801
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.75
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    0.83
+# CHECK-NEXT: IPC:               0.56
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeE .    .    .    .   ld1r	{ v1.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     . DE .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .  DeeE   .    .    .   ld1r	{ v1.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     .    DE   .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .DeeeeeE  .    .   ld2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,5]     .    .    .DE  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    . DeeeE   .   ld2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,7]     .    .    .    DE   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .DeeeE   ld2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,9]     .    .    .    .   DE   add	x0, x27, #1
+# CHECK:      [0,0]     DeeE .    .    .  .   ld1r	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     . DE .    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  DeeE   .    .  .   ld1r	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .    DE   .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .DeeeE    .  .   ld2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,5]     .    .   DE    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    DeeeE.  .   ld2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,7]     .    .    .  DE.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .   DeeeE   ld2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,9]     .    .    .    . DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2036,28 +2036,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2801
+# CHECK-NEXT: Total Cycles:      2001
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.54
-# CHECK-NEXT: IPC:               0.36
-# CHECK-NEXT: Block RThroughput: 9.0
+# CHECK-NEXT: uOps Per Cycle:    0.75
+# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345678
-
-# CHECK:      [0,0]     DeeeeeE   .    .    .    .  .   ld2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,1]     .    DE   .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .    .DeeeE    .    .    .  .   ld2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,3]     .    .   DE    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .    DeeeeeE   .    .  .   ld2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,5]     .    .    .    DE   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    .    .DeeeeeE  .  .   ld2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,7]     .    .    .    .    .DE  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .    . DeeeeeE   ld2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .    .    .    . DE   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0
+
+# CHECK:      [0,0]     DeeeE.    .    .    .   ld2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,1]     .  DE.    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .   DeeeE .    .    .   ld2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,3]     .    . DE .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .  DeeeE  .    .   ld2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,5]     .    .    .DE  .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    . DeeeE   .   ld2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,7]     .    .    .    DE   .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .    .DeeeE   ld2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .    .   DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2082,28 +2082,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2401
+# CHECK-NEXT: Total Cycles:      2001
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.62
-# CHECK-NEXT: IPC:               0.42
+# CHECK-NEXT: uOps Per Cycle:    0.75
+# CHECK-NEXT: IPC:               0.50
 # CHECK-NEXT: Block RThroughput: 7.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          01234
+# CHECK-NEXT: Index     0123456789          0
 
-# CHECK:      [0,0]     DeeeE.    .    .    .   .   ld2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     .  DE.    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .   DeeeE .    .    .   .   ld2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .    . DE .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .  DeeeeeE.    .   .   ld2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     .    .    .  DE.    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    .   DeeeE .   .   ld2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .    .    .    . DE .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .  DeeeeeE   ld2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .    .    .  DE   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeE.    .    .    .   ld2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     .  DE.    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .   DeeeE .    .    .   ld2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .    . DE .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .  DeeeE  .    .   ld2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .    .    .DE  .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    . DeeeE   .   ld2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .    .    DE   .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .    .DeeeE   ld2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .    .   DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2128,28 +2128,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2201
+# CHECK-NEXT: Total Cycles:      2001
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.68
-# CHECK-NEXT: IPC:               0.45
-# CHECK-NEXT: Block RThroughput: 6.0
+# CHECK-NEXT: uOps Per Cycle:    0.75
+# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012
+# CHECK-NEXT: Index     0123456789          0
 
-# CHECK:      [0,0]     DeeeeeE   .    .    . .   ld2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     .    DE   .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .    .DeeeE    .    . .   ld2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: [0,3]     .    .   DE    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .    DeeeE.    . .   ld2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: [0,5]     .    .    .  DE.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    .   DeeeE . .   ld2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .    .    .    . DE . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .  DeeeE   ld2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .    .    .DE   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeE.    .    .    .   ld2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     .  DE.    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .   DeeeE .    .    .   ld2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: [0,3]     .    . DE .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .  DeeeE  .    .   ld2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: [0,5]     .    .    .DE  .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    . DeeeE   .   ld2	{ v1.b, v2.b }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .    .    .    DE   .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .    .DeeeE   ld2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .    .   DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2180,7 +2180,7 @@ add x0, x27, 1
 # CHECK:      Dispatch Width:    3
 # CHECK-NEXT: uOps Per Cycle:    0.75
 # CHECK-NEXT: IPC:               0.50
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
@@ -2220,28 +2220,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2001
+# CHECK-NEXT: Total Cycles:      1801
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.75
-# CHECK-NEXT: IPC:               0.50
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: uOps Per Cycle:    0.83
+# CHECK-NEXT: IPC:               0.56
+# CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeE.    .    .    .   ld2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: [0,1]     .  DE.    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .   DeeeE .    .    .   ld2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: [0,3]     .    . DE .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .  DeeeE  .    .   ld2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .    .    .DE  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    . DeeeE   .   ld2r	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,7]     .    .    .    DE   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .DeeeE   ld2r	{ v1.2d, v2.2d }, [x27], #16
-# CHECK-NEXT: [0,9]     .    .    .    .   DE   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeE.    .    .  .   ld2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: [0,1]     .  DE.    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .   DeeeE .    .  .   ld2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: [0,3]     .    . DE .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .  DeeeE  .  .   ld2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .    .    .DE  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    . DeeE  .   ld2r	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,7]     .    .    .   DE  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .    DeeE   ld2r	{ v1.2d, v2.2d }, [x27], #16
+# CHECK-NEXT: [0,9]     .    .    .    . DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2266,28 +2266,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2001
+# CHECK-NEXT: Total Cycles:      1501
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.75
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    1.00
+# CHECK-NEXT: IPC:               0.67
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0
+# CHECK-NEXT:                     012345
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeE.    .    .    .   ld2r	{ v1.2s, v2.2s }, [x27], #8
-# CHECK-NEXT: [0,1]     .  DE.    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .   DeeeE .    .    .   ld2r	{ v1.4h, v2.4h }, [x27], #4
-# CHECK-NEXT: [0,3]     .    . DE .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .  DeeeE  .    .   ld2r	{ v1.4s, v2.4s }, [x27], #8
-# CHECK-NEXT: [0,5]     .    .    .DE  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    . DeeeE   .   ld2r	{ v1.8b, v2.8b }, [x27], #2
-# CHECK-NEXT: [0,7]     .    .    .    DE   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .DeeeE   ld2r	{ v1.8h, v2.8h }, [x27], #4
-# CHECK-NEXT: [0,9]     .    .    .    .   DE   add	x0, x27, #1
+# CHECK:      [0,0]     DeeE .    .    .   ld2r	{ v1.2s, v2.2s }, [x27], #8
+# CHECK-NEXT: [0,1]     . DE .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  DeeE   .    .   ld2r	{ v1.4h, v2.4h }, [x27], #4
+# CHECK-NEXT: [0,3]     .    DE   .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .DeeE.    .   ld2r	{ v1.4s, v2.4s }, [x27], #8
+# CHECK-NEXT: [0,5]     .    .  DE.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .   DeeE  .   ld2r	{ v1.8b, v2.8b }, [x27], #2
+# CHECK-NEXT: [0,7]     .    .    .DE  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    . DeeE   ld2r	{ v1.8h, v2.8h }, [x27], #4
+# CHECK-NEXT: [0,9]     .    .    .   DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2312,28 +2312,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2001
+# CHECK-NEXT: Total Cycles:      1501
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.75
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    1.00
+# CHECK-NEXT: IPC:               0.67
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0
+# CHECK-NEXT:                     012345
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeE.    .    .    .   ld2r	{ v1.16b, v2.16b }, [x27], #2
-# CHECK-NEXT: [0,1]     .  DE.    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .   DeeeE .    .    .   ld2r	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,3]     .    . DE .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .  DeeeE  .    .   ld2r	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,5]     .    .    .DE  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    . DeeeE   .   ld2r	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,7]     .    .    .    DE   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .DeeeE   ld2r	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .    .   DE   add	x0, x27, #1
+# CHECK:      [0,0]     DeeE .    .    .   ld2r	{ v1.16b, v2.16b }, [x27], #2
+# CHECK-NEXT: [0,1]     . DE .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  DeeE   .    .   ld2r	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,3]     .    DE   .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .DeeE.    .   ld2r	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,5]     .    .  DE.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .   DeeE  .   ld2r	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .    .DE  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    . DeeE   ld2r	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .   DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2358,28 +2358,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2101
+# CHECK-NEXT: Total Cycles:      1701
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.71
-# CHECK-NEXT: IPC:               0.48
-# CHECK-NEXT: Block RThroughput: 5.5
+# CHECK-NEXT: uOps Per Cycle:    0.88
+# CHECK-NEXT: IPC:               0.59
+# CHECK-NEXT: Block RThroughput: 7.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          01
+# CHECK-NEXT:                     01234567
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeE.    .    .    ..   ld2r	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,1]     .  DE.    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .   DeeeE .    .    ..   ld2r	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,3]     .    . DE .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .  DeeeE  .    ..   ld2r	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,5]     .    .    .DE  .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    . DeeeE   ..   ld2r	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,7]     .    .    .    DE   ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .DeeeeE   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,9]     .    .    .    .    DE   add	x0, x27, #1
+# CHECK:      [0,0]     DeeE .    .    . .   ld2r	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,1]     . DE .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  DeeE   .    . .   ld2r	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     .    DE   .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .DeeE.    . .   ld2r	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,5]     .    .  DE.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .   DeeE  . .   ld2r	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .    .DE  . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    . DeeeeE   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,9]     .    .    .    .DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2410,7 +2410,7 @@ add x0, x27, 1
 # CHECK:      Dispatch Width:    3
 # CHECK-NEXT: uOps Per Cycle:    0.60
 # CHECK-NEXT: IPC:               0.40
-# CHECK-NEXT: Block RThroughput: 7.5
+# CHECK-NEXT: Block RThroughput: 15.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
@@ -2456,7 +2456,7 @@ add x0, x27, 1
 # CHECK:      Dispatch Width:    3
 # CHECK-NEXT: uOps Per Cycle:    0.60
 # CHECK-NEXT: IPC:               0.40
-# CHECK-NEXT: Block RThroughput: 7.5
+# CHECK-NEXT: Block RThroughput: 15.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
@@ -2496,28 +2496,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2301
+# CHECK-NEXT: Total Cycles:      2501
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.65
-# CHECK-NEXT: IPC:               0.43
-# CHECK-NEXT: Block RThroughput: 6.5
+# CHECK-NEXT: uOps Per Cycle:    0.60
+# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: Block RThroughput: 15.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
-
-# CHECK:      [0,0]     DeeeeE    .    .    .  .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     .   DE    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .    DeeeeE    .    .  .   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     .    .   DE    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .    DeeeeE    .  .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     .    .    .   DE    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    .    DeeeE.  .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: [0,7]     .    .    .    .  DE.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .   DeeeE   ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: [0,9]     .    .    .    .    . DE   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          012345
+
+# CHECK:      [0,0]     DeeeeE    .    .    .    .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     .   DE    .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .    DeeeeE    .    .    .   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .    .   DE    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .    DeeeeE    .    .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     .    .    .   DE    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    .    DeeeeE    .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: [0,7]     .    .    .    .   DE    .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .    .    DeeeeE   ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: [0,9]     .    .    .    .    .   DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2542,28 +2542,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2001
+# CHECK-NEXT: Total Cycles:      2501
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.75
-# CHECK-NEXT: IPC:               0.50
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: uOps Per Cycle:    0.60
+# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: Block RThroughput: 15.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0
+# CHECK-NEXT: Index     0123456789          012345
 
-# CHECK:      [0,0]     DeeeE.    .    .    .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: [0,1]     .  DE.    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .   DeeeE .    .    .   ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: [0,3]     .    . DE .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .  DeeeE  .    .   ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: [0,5]     .    .    .DE  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    . DeeeE   .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: [0,7]     .    .    .    DE   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .DeeeE   ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .    .   DE   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeE    .    .    .    .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: [0,1]     .   DE    .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .    DeeeeE    .    .    .   ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: [0,3]     .    .   DE    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .    DeeeeE    .    .   ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
+# CHECK-NEXT: [0,5]     .    .    .   DE    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    .    DeeeeE    .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: [0,7]     .    .    .    .   DE    .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .    .    DeeeeE   ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .    .    .   DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2588,28 +2588,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2001
+# CHECK-NEXT: Total Cycles:      2501
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.75
-# CHECK-NEXT: IPC:               0.50
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: uOps Per Cycle:    0.60
+# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: Block RThroughput: 15.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0
+# CHECK-NEXT: Index     0123456789          012345
 
-# CHECK:      [0,0]     DeeeE.    .    .    .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: [0,1]     .  DE.    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .   DeeeE .    .    .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: [0,3]     .    . DE .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .  DeeeE  .    .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .    .    .DE  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    . DeeeE   .   ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: [0,7]     .    .    .    DE   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .DeeeE   ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .    .   DE   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeE    .    .    .    .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: [0,1]     .   DE    .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .    DeeeeE    .    .    .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: [0,3]     .    .   DE    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .    DeeeeE    .    .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .    .    .   DE    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    .    DeeeeE    .   ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: [0,7]     .    .    .    .   DE    .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .    .    DeeeeE   ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .    .    .   DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2640,7 +2640,7 @@ add x0, x27, 1
 # CHECK:      Dispatch Width:    3
 # CHECK-NEXT: uOps Per Cycle:    0.75
 # CHECK-NEXT: IPC:               0.50
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
@@ -2686,7 +2686,7 @@ add x0, x27, 1
 # CHECK:      Dispatch Width:    3
 # CHECK-NEXT: uOps Per Cycle:    0.75
 # CHECK-NEXT: IPC:               0.50
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
@@ -2732,7 +2732,7 @@ add x0, x27, 1
 # CHECK:      Dispatch Width:    3
 # CHECK-NEXT: uOps Per Cycle:    0.75
 # CHECK-NEXT: IPC:               0.50
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
@@ -2772,28 +2772,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2801
+# CHECK-NEXT: Total Cycles:      2401
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.54
-# CHECK-NEXT: IPC:               0.36
-# CHECK-NEXT: Block RThroughput: 9.0
+# CHECK-NEXT: uOps Per Cycle:    0.62
+# CHECK-NEXT: IPC:               0.42
+# CHECK-NEXT: Block RThroughput: 15.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345678
-
-# CHECK:      [0,0]     DeeeE.    .    .    .    .  .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     .  DE.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .   DeeeeeE    .    .    .  .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,3]     .    .   DE    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .    DeeeeeE   .    .  .   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,5]     .    .    .    DE   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    .    .DeeeeeE  .  .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,7]     .    .    .    .    .DE  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .    . DeeeeeE   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,9]     .    .    .    .    .    . DE   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          01234
+
+# CHECK:      [0,0]     DeeeE.    .    .    .   .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     .  DE.    .    .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .   DeeeeE.    .    .   .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,3]     .    .  DE.    .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .   DeeeeE.    .   .   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,5]     .    .    .  DE.    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    .   DeeeeE.   .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,7]     .    .    .    .  DE.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .    .   DeeeeE   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,9]     .    .    .    .    .  DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2818,28 +2818,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3001
+# CHECK-NEXT: Total Cycles:      2501
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.50
-# CHECK-NEXT: IPC:               0.33
-# CHECK-NEXT: Block RThroughput: 10.0
+# CHECK-NEXT: uOps Per Cycle:    0.60
+# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: Block RThroughput: 16.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345
 
-# CHECK:      [0,0]     DeeeeeE   .    .    .    .    .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,1]     .    DE   .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .    .DeeeeeE  .    .    .    .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,3]     .    .    .DE  .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .    . DeeeeeE .    .    .   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,5]     .    .    .    . DE .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    .    .  DeeeeeE.    .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,7]     .    .    .    .    .  DE.    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .    .   DeeeeeE   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .    .    .    .   DE   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeE    .    .    .    .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,1]     .   DE    .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .    DeeeeE    .    .    .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,3]     .    .   DE    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .    DeeeeE    .    .   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,5]     .    .    .   DE    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    .    DeeeeE    .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .    .    .   DE    .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .    .    DeeeeE   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .    .    .   DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2864,28 +2864,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3001
+# CHECK-NEXT: Total Cycles:      2501
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.50
-# CHECK-NEXT: IPC:               0.33
-# CHECK-NEXT: Block RThroughput: 10.0
+# CHECK-NEXT: uOps Per Cycle:    0.60
+# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: Block RThroughput: 15.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345
 
-# CHECK:      [0,0]     DeeeeeE   .    .    .    .    .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,1]     .    DE   .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .    .DeeeeeE  .    .    .    .   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,3]     .    .    .DE  .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .    . DeeeeeE .    .    .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,5]     .    .    .    . DE .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    .    .  DeeeeeE.    .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,7]     .    .    .    .    .  DE.    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .    .   DeeeeeE   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .    .    .    .   DE   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeE    .    .    .    .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,1]     .   DE    .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .    DeeeeE    .    .    .   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     .    .   DE    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .    DeeeeE    .    .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,5]     .    .    .   DE    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    .    DeeeeE    .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .    .    .   DE    .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .    .    DeeeeE   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .    .    .   DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2910,28 +2910,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2001
+# CHECK-NEXT: Total Cycles:      3001
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.75
-# CHECK-NEXT: IPC:               0.50
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: uOps Per Cycle:    0.50
+# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: Block RThroughput: 20.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0
+# CHECK-NEXT:                     0123456789          0
+# CHECK-NEXT: Index     0123456789          0123456789
 
-# CHECK:      [0,0]     DeeeE.    .    .    .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: [0,1]     .  DE.    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .   DeeeE .    .    .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: [0,3]     .    . DE .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .  DeeeE  .    .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .    .    .DE  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    . DeeeE   .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: [0,7]     .    .    .    DE   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .DeeeE   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: [0,9]     .    .    .    .   DE   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeE   .    .    .    .    .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: [0,1]     .    DE   .    .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .    .DeeeeeE  .    .    .    .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: [0,3]     .    .    .DE  .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .    . DeeeeeE .    .    .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .    .    .    . DE .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    .    .  DeeeeeE.    .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+# CHECK-NEXT: [0,7]     .    .    .    .    .  DE.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .    .    .   DeeeeeE   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: [0,9]     .    .    .    .    .    .   DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2956,28 +2956,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2001
+# CHECK-NEXT: Total Cycles:      3001
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.75
-# CHECK-NEXT: IPC:               0.50
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: uOps Per Cycle:    0.50
+# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: Block RThroughput: 20.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0
+# CHECK-NEXT:                     0123456789          0
+# CHECK-NEXT: Index     0123456789          0123456789
 
-# CHECK:      [0,0]     DeeeE.    .    .    .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: [0,1]     .  DE.    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .   DeeeE .    .    .   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .    . DE .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .  DeeeE  .    .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: [0,5]     .    .    .DE  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    . DeeeE   .   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: [0,7]     .    .    .    DE   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .DeeeE   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .    .   DE   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeE   .    .    .    .    .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: [0,1]     .    DE   .    .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .    .DeeeeeE  .    .    .    .   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .    .    .DE  .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .    . DeeeeeE .    .    .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: [0,5]     .    .    .    . DE .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    .    .  DeeeeeE.    .   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+# CHECK-NEXT: [0,7]     .    .    .    .    .  DE.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .    .    .   DeeeeeE   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .    .    .    .   DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3002,28 +3002,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2001
+# CHECK-NEXT: Total Cycles:      2401
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.75
-# CHECK-NEXT: IPC:               0.50
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: uOps Per Cycle:    0.62
+# CHECK-NEXT: IPC:               0.42
+# CHECK-NEXT: Block RThroughput: 14.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0
+# CHECK-NEXT: Index     0123456789          01234
 
-# CHECK:      [0,0]     DeeeE.    .    .    .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: [0,1]     .  DE.    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .   DeeeE .    .    .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .    . DE .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .  DeeeE  .    .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,5]     .    .    .DE  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    . DeeeE   .   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
-# CHECK-NEXT: [0,7]     .    .    .    DE   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .DeeeE   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
-# CHECK-NEXT: [0,9]     .    .    .    .   DE   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeE   .    .    .   .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: [0,1]     .    DE   .    .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .    .DeeeeeE  .    .   .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .    .    .DE  .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .    . DeeeE   .   .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,5]     .    .    .    DE   .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    .    .DeeeE   .   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
+# CHECK-NEXT: [0,7]     .    .    .    .   DE   .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .    .    DeeeE   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
+# CHECK-NEXT: [0,9]     .    .    .    .    .  DE   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3054,7 +3054,7 @@ add x0, x27, 1
 # CHECK:      Dispatch Width:    3
 # CHECK-NEXT: uOps Per Cycle:    0.75
 # CHECK-NEXT: IPC:               0.50
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
@@ -3100,7 +3100,7 @@ add x0, x27, 1
 # CHECK:      Dispatch Width:    3
 # CHECK-NEXT: uOps Per Cycle:    0.75
 # CHECK-NEXT: IPC:               0.50
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
@@ -3146,7 +3146,7 @@ add x0, x27, 1
 # CHECK:      Dispatch Width:    3
 # CHECK-NEXT: uOps Per Cycle:    0.94
 # CHECK-NEXT: IPC:               0.56
-# CHECK-NEXT: Block RThroughput: 5.7
+# CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012345678
diff --git a/llvm/tools/llvm-debuginfo-analyzer/README.md b/llvm/tools/llvm-debuginfo-analyzer/README.md
index 28c4be6f73229..6b2d392c57d5d 100644
--- a/llvm/tools/llvm-debuginfo-analyzer/README.md
+++ b/llvm/tools/llvm-debuginfo-analyzer/README.md
@@ -3,14 +3,14 @@
 These are the notes collected during the development, review and test.
 They describe limitations, known issues and future work.
 
-### Remove the use of macros in ``LVReader.h`` that describe the ``bumpallocators``.
+## Remove the use of macros in ``LVReader.h`` that describe the ``bumpallocators``.
 **[D137933](https://reviews.llvm.org/D137933#inline-1389904)**
 
 Use a standard (or LLVM) ``map`` with ``typeinfo`` (would need a specialization
 to expose equality and hasher) for the allocators and the creation
 functions could be a function template.
 
-### Use a **lit test** instead of a **unit test** for the **logical readers**.
+## Use a **lit test** instead of a **unit test** for the **logical readers**.
 **[D125783](https://reviews.llvm.org/D125783#inline-1324376)**
 
 As the ``DebugInfoLogicalView`` library is sufficiently exposed via the
@@ -28,7 +28,7 @@ llvm-project/llvm/test/DebugInfo/LogicalView/CodeViewReader.test
 llvm-project/llvm/test/DebugInfo/LogicalView/DWARFReader.test
 ```
 
-### Eliminate calls to ``getInputFileDirectory()`` in the ``unittests``.
+## Eliminate calls to ``getInputFileDirectory()`` in the ``unittests``.
 **[D125783](https://reviews.llvm.org/D125783#inline-1324359)**
 
 Rewrite the unittests ``ReaderTest`` and ``CodeViewReaderTest`` to eliminate
@@ -38,37 +38,37 @@ the call:
 ```
 as use of that call is discouraged.
 
-### Fix mismatch between ``%d/%x`` format strings and ``uint64_t`` type.
+## Fix mismatch between ``%d/%x`` format strings and ``uint64_t`` type.
 **[D137400](https://reviews.llvm.org/D137400) / [58758](https://github.com/llvm/llvm-project/issues/58758)**
 
 Incorrect printing of ``uint64_t`` on ``32-bit`` platforms.
 Add the ``PRIx64`` specifier to the printing code (``format()``).
 
-### Remove ``LVScope::Children`` container.
+## Remove ``LVScope::Children`` container.
 **[D137933](https://reviews.llvm.org/D137933#inline-1373902)**
 
 Use a **chaining iterator** over the other containers rather than keep a
 separate container ``Children`` that mirrors their contents.
 
-### Use ``TableGen`` for command line options.
+## Use ``TableGen`` for command line options.
 **[D125777](https://reviews.llvm.org/D125777#inline-1291801)**
 
 The current trend is to use ``TableGen`` for command-line options in tools.
 Change command line options to use ``tablegen`` as many other LLVM tools.
 
-### ``LVDoubleMap`` to return ``optional<ValueType>`` instead of ``null pointer``.
+## ``LVDoubleMap`` to return ``optional<ValueType>`` instead of ``null pointer``.
 **[D125783](https://reviews.llvm.org/D125783#inline-1294164)**
 
 The more idiomatic LLVM way to handle this would be to have ``find``
 return ``Optional<ValueType>``.
 
-### Pass references instead of pointers (**Comparison functions**).
+## Pass references instead of pointers (**Comparison functions**).
 **[D125782](https://reviews.llvm.org/D125782#inline-1293920)**
 
 In the **comparison functions**, pass references instead of pointers (when
 pointers cannot be null).
 
-### Use ``StringMap`` where possible.
+## Use ``StringMap`` where possible.
 **[D125783](https://reviews.llvm.org/D125783#inline-1294211)**
 
 LLVM has a ``StringMap`` class that is advertised as more efficient than
@@ -78,7 +78,7 @@ because the key is not a ``std::string``.
 Replace the use of ``std::map<std::string, ValueType>`` with ``StringMap``.
 One specific case is the ``LVSymbolNames`` definitions.
 
-### Calculate unique offset for CodeView elements.
+## Calculate unique offset for CodeView elements.
 In order to have the same logical functionality as the DWARF reader, such
 as:
 
@@ -88,7 +88,7 @@ as:
 The logical elements must have an unique offset (similar like the DWARF
 ``DIE`` offset).
 
-### Move ``initializeFileAndStringTables`` to the CodeView Library.
+## Move ``initializeFileAndStringTables`` to the CodeView Library.
 There is some code in the CodeView reader that was extracted/adapted
 from ``tools/llvm-readobj/COFFDumper.cpp`` that can be moved to the CodeView
 library.
@@ -96,18 +96,18 @@ library.
 We had a similar case with code shared with ``llvm-pdbutil`` that was moved
 to the PDB library: **[D122226](https://reviews.llvm.org/D122226)**
 
-### Move ``getSymbolKindName`` and ``formatRegisterId`` to the CodeView Library.
+## Move ``getSymbolKindName`` and ``formatRegisterId`` to the CodeView Library.
 There is some code in the CodeView reader that was extracted/adapted
 from ``lib/DebugInfo/CodeView/SymbolDumper.cpp`` that can be used.
 
-### Use of ``std::unordered_set`` instead of ``std::set``.
+## Use of ``std::unordered_set`` instead of ``std::set``.
 **[D125784](https://reviews.llvm.org/D125784#inline-1221421)**
 
 Replace the ``std::set`` usage for ``DeducedScopes``, ``UnresolvedScopes`` and
 ``IdentifiedNamespaces`` with ``std::unordered_set`` and get the benefit
 of the O(1) while inserting/searching, as the order is not important.
 
-### Optimize ``LVNamespaceDeduction::find`` funtion.
+## Optimize ``LVNamespaceDeduction::find`` funtion.
 **[D125784](https://reviews.llvm.org/D125784#inline-1296195)**
 
 Optimize the ``find`` method to use the proposed code:
@@ -120,11 +120,11 @@ Optimize the ``find`` method to use the proposed code:
   LVStringRefs::size_type FirstNonNamespace = std::distance(Components.begin(), Iter);
 ```
 
-### Move all the printing support to a common module.
+## Move all the printing support to a common module.
 Factor out printing functionality from the logical elements into a
 common module.
 
-### Refactor ``LVBinaryReader::processLines``.
+## Refactor ``LVBinaryReader::processLines``.
 **[D125783](https://reviews.llvm.org/D125783#inline-1246155) /
 [D137156](https://reviews.llvm.org/D137156)**
 
@@ -137,13 +137,13 @@ we will allocate those logical lines to their logical scopes.
 Consider the case when any of those lines become orphans, causing
 incorrect scope parent for disassembly or line records.
 
-### Add support for ``-ffunction-sections``.
+## Add support for ``-ffunction-sections``.
 **[D125783](https://reviews.llvm.org/D125783#inline-1295012)**
 
 Only linked executables are handled. It does not support relocatable
 files compiled with ``-ffunction-sections``.
 
-### Add support for DWARF v5 `.debug_names` section / CodeView public symbols stream.
+## Add support for DWARF v5 `.debug_names` section / CodeView public symbols stream.
 **[D125783](https://reviews.llvm.org/D125783#inline-1294142)**
 
 The DWARF and CodeView readers use the public names information to create
@@ -155,16 +155,16 @@ the debug information.
 If the object file supports the above section names and stream, use them
 to create the public names.
 
-### Add support for some extra DWARF locations.
+## Add support for some extra DWARF locations.
 The following DWARF debug location operands are not supported:
 
 * `DW_OP_const_type`
 * `DW_OP_entry_value`
 * `DW_OP_implicit_value`
 
-### Add support for additional binary formats.
+## Add support for additional binary formats.
 * Extended COFF (`XCOFF`)
 
-### Add support for ``JSON`` or ``YAML``
+## Add support for ``JSON`` or ``YAML``
 The logical view uses its own and non-standard free form text when
 displaying information on logical elements.
diff --git a/llvm/unittests/IR/FunctionTest.cpp b/llvm/unittests/IR/FunctionTest.cpp
index 8ed76999dc13d..82b67c6e629b4 100644
--- a/llvm/unittests/IR/FunctionTest.cpp
+++ b/llvm/unittests/IR/FunctionTest.cpp
@@ -644,4 +644,17 @@ TEST(FunctionTest, LLVMGetOrInsertFunction) {
   EXPECT_EQ(FuncRef, FuncRef2);
 }
 
+TEST(FunctionTest, NoIPAInterposable) {
+  LLVMContext Ctx;
+  std::unique_ptr<Module> M = parseIR(Ctx, R"(
+    define void @foo() { bb1: ret void }
+    define void @bar() #0 { bb1: ret void }
+    attributes #0 = { noipa }
+  )");
+  EXPECT_FALSE(M->getFunction("foo")->isInterposable());
+  EXPECT_TRUE(M->getFunction("foo")->isDefinitionExact());
+  EXPECT_TRUE(M->getFunction("bar")->isInterposable());
+  EXPECT_FALSE(M->getFunction("bar")->isDefinitionExact());
+}
+
 } // end namespace
diff --git a/llvm/utils/emacs/llvm-mode.el b/llvm/utils/emacs/llvm-mode.el
index 7dc1c28489839..6ee5a30e5e54c 100644
--- a/llvm/utils/emacs/llvm-mode.el
+++ b/llvm/utils/emacs/llvm-mode.el
@@ -35,7 +35,7 @@
    `(,(regexp-opt
        '("alwaysinline" "argmemonly" "allocsize" "builtin" "cold" "convergent" "dereferenceable" "dereferenceable_or_null" "hot" "immarg" "inaccessiblememonly"
          "inaccessiblemem_or_argmemonly" "inalloca" "inlinehint" "jumptable" "minsize" "mustprogress" "naked" "nobuiltin" "nonnull" "nocapture"
-         "nocallback" "nocf_check" "noduplicate" "noext" "nofree" "noimplicitfloat" "noinline" "nomerge" "nonlazybind" "noprofile" "noredzone" "noreturn"
+         "nocallback" "nocf_check" "noduplicate" "noext" "nofree" "noimplicitfloat" "noinline" "noipa" "nomerge" "nonlazybind" "noprofile" "noredzone" "noreturn"
          "norecurse" "nosync" "noundef" "nounwind" "nosanitize_bounds" "nosanitize_coverage" "null_pointer_is_valid" "optdebug" "optforfuzzing" "optnone" "optsize" "preallocated" "readnone" "readonly" "returned" "returns_twice"
          "shadowcallstack" "signext" "speculatable" "speculative_load_hardening" "ssp" "sspreq" "sspstrong" "safestack" "sanitize_address" "sanitize_alloc_token" "sanitize_hwaddress" "sanitize_memtag"
          "sanitize_thread" "sanitize_memory" "strictfp" "swifterror" "uwtable" "vscale_range" "willreturn" "writeonly" "zeroext") 'symbols) . font-lock-constant-face)
diff --git a/llvm/utils/kate/llvm.xml b/llvm/utils/kate/llvm.xml
index c66547d9121ce..0291ed32b3f6c 100644
--- a/llvm/utils/kate/llvm.xml
+++ b/llvm/utils/kate/llvm.xml
@@ -104,6 +104,7 @@
       <item> nofree </item>
       <item> noimplicitfloat </item>
       <item> noinline </item>
+      <item> noipa </item>
       <item> nomerge </item>
       <item> nooutline </item>
       <item> noprofile </item>
diff --git a/llvm/utils/vim/syntax/llvm.vim b/llvm/utils/vim/syntax/llvm.vim
index d921e8bd1e401..bb805dff10a7b 100644
--- a/llvm/utils/vim/syntax/llvm.vim
+++ b/llvm/utils/vim/syntax/llvm.vim
@@ -132,6 +132,7 @@ syn keyword llvmKeyword
       \ nofree
       \ noimplicitfloat
       \ noinline
+      \ noipa
       \ nomerge
       \ nooutline
       \ nonlazybind
diff --git a/llvm/utils/vscode/llvm/syntaxes/ll.tmLanguage.yaml b/llvm/utils/vscode/llvm/syntaxes/ll.tmLanguage.yaml
index 06d2e43bb3c2e..6a6caccb350a5 100644
--- a/llvm/utils/vscode/llvm/syntaxes/ll.tmLanguage.yaml
+++ b/llvm/utils/vscode/llvm/syntaxes/ll.tmLanguage.yaml
@@ -229,6 +229,7 @@ patterns:
             \\bnofree\\b|\
             \\bnoimplicitfloat\\b|\
             \\bnoinline\\b|\
+            \\bnoipa\\b|\
             \\bnomerge\\b|\
             \\bnooutline\\b|\
             \\bnonlazybind\\b|\
diff --git a/mlir/include/mlir/Conversion/Passes.h b/mlir/include/mlir/Conversion/Passes.h
index 82c7670296e52..0ea248df3fee4 100644
--- a/mlir/include/mlir/Conversion/Passes.h
+++ b/mlir/include/mlir/Conversion/Passes.h
@@ -61,6 +61,7 @@
 #include "mlir/Conversion/OpenACCToSCF/ConvertOpenACCToSCF.h"
 #include "mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h"
 #include "mlir/Conversion/PDLToPDLInterp/PDLToPDLInterp.h"
+#include "mlir/Conversion/RaiseWasm/RaiseWasmMLIR.h"
 #include "mlir/Conversion/ReconcileUnrealizedCasts/ReconcileUnrealizedCasts.h"
 #include "mlir/Conversion/SCFToControlFlow/SCFToControlFlow.h"
 #include "mlir/Conversion/SCFToEmitC/SCFToEmitC.h"
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index d68afbdc2a9ee..b2c608a5c5258 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -1724,6 +1724,19 @@ def ConvertVectorToAMX : Pass<"convert-vector-to-amx"> {
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// RaiseWasmMLIR
+//===----------------------------------------------------------------------===//
+
+def RaiseWasmMLIR : Pass<"raise-wasm-mlir"> {
+  let summary = "Convert Wasm dialect to a group of dialect as a bridge to LLVM MLIR conversion";
+  let dependentDialects = [
+    "func::FuncDialect", "arith::ArithDialect", "cf::ControlFlowDialect",
+    "memref::MemRefDialect", "vector::VectorDialect", "wasmssa::WasmSSADialect",
+    "math::MathDialect"
+  ];
+}
+
 //===----------------------------------------------------------------------===//
 // XeVMToLLVM
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Conversion/RaiseWasm/RaiseWasmMLIR.h b/mlir/include/mlir/Conversion/RaiseWasm/RaiseWasmMLIR.h
new file mode 100644
index 0000000000000..048eeb3149cf7
--- /dev/null
+++ b/mlir/include/mlir/Conversion/RaiseWasm/RaiseWasmMLIR.h
@@ -0,0 +1,29 @@
+//===- RaiseWasmMLIR.h - Convert wasm to standard dialects ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_CONVERSION_RAISEWASM_RAISEWASMMLIR_H
+#define MLIR_CONVERSION_RAISEWASM_RAISEWASMMLIR_H
+
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Transforms/DialectConversion.h"
+
+namespace mlir {
+class Pass;
+class RewritePatternSet;
+
+#define GEN_PASS_DECL_RAISEWASMMLIR
+#include "mlir/Conversion/Passes.h.inc"
+
+/// Collect a set of patterns to convert from the Wasm dialect to standard
+/// dialects.
+void populateRaiseWasmMLIRConversionPatterns(TypeConverter &,
+                                             RewritePatternSet &);
+
+} // namespace mlir
+
+#endif // MLIR_CONVERSION_RAISEWASM_RAISEWASMMLIR_H
diff --git a/mlir/include/mlir/Dialect/OpenACC/Transforms/Passes.td b/mlir/include/mlir/Dialect/OpenACC/Transforms/Passes.td
index e35e38adceb02..b52dee12d390a 100644
--- a/mlir/include/mlir/Dialect/OpenACC/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/OpenACC/Transforms/Passes.td
@@ -193,6 +193,20 @@ def ACCEmitRemarksLoop : Pass<"acc-emit-remarks-loop", "mlir::func::FuncOp"> {
   ];
 }
 
+def ACCEmitRemarksPrivate
+    : Pass<"acc-emit-remarks-private", "mlir::func::FuncOp"> {
+  let summary = "Emit OpenACC private and firstprivate remarks";
+  let description = [{
+    This pass emits optimization remarks describing the private and firstprivate
+    variables associated with OpenACC compute and loop constructs.
+
+    The pass walks compute constructs and `acc.loop` operations and, for each,
+    reports the implicit and explicit `private` and `firstprivate` variables it
+    carries.
+  }];
+  let dependentDialects = ["mlir::acc::OpenACCDialect"];
+}
+
 def ACCLoopTiling : Pass<"acc-loop-tiling", "mlir::func::FuncOp"> {
   let summary = "Tile OpenACC loops with tile clauses";
   let description = [{
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h
index d7bccb133e02c..c61e9415327f0 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h
@@ -24,6 +24,7 @@
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
+#include "llvm/Frontend/OpenMP/OMPConstants.h"
 #include "llvm/Frontend/OpenMP/OMPDeviceConstants.h"
 
 #define GET_TYPEDEF_CLASSES
@@ -51,6 +52,11 @@
 namespace mlir::omp {
 /// Find the omp.new_cli, generator, and consumer of a canonical loop info.
 std::tuple<NewCliOp, OpOperand *, OpOperand *> decodeCli(mlir::Value cli);
+
+/// Convert a proc_bind kind from the LLVM frontend enum to the corresponding
+/// OpenMP dialect enum. The LLVM 'default' and 'unknown' kinds have no dialect
+/// counterpart and are not valid inputs.
+ClauseProcBindKind convertProcBindKind(llvm::omp::ProcBindKind kind);
 } // namespace mlir::omp
 
 #endif // MLIR_DIALECT_OPENMP_OPENMPDIALECT_H_
diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVCLOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVCLOps.td
index 617b41893c216..7c1ca28f728e9 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVCLOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVCLOps.td
@@ -319,6 +319,27 @@ def SPIRV_CLCeilOp : SPIRV_CLUnaryArithmeticOp<"ceil", 12, SPIRV_Float> {
 
 // -----
 
+def SPIRV_CLCopysignOp : SPIRV_CLBinaryArithmeticOp<"copysign", 13, SPIRV_Float> {
+  let summary = "Computes the value with the magnitude of x and the sign of y.";
+
+  let description = [{
+    Result Type, x and y must be floating-point or vector(2,3,4,8,16) of
+    floating-point values.
+
+    All of the operands, including the Result Type operand, must be of the
+    same type.
+
+    #### Example:
+
+    ```mlir
+    %2 = spirv.CL.copysign %0, %1 : f32
+    %3 = spirv.CL.copysign %0, %1 : vector<3xf16>
+    ```
+  }];
+}
+
+// -----
+
 def SPIRV_CLCosOp : SPIRV_CLUnaryArithmeticOp<"cos", 14, SPIRV_Float> {
   let summary = "Compute the cosine of x radians.";
 
@@ -562,6 +583,73 @@ def SPIRV_CLFMinOp : SPIRV_CLBinaryArithmeticOp<"fmin", 28, SPIRV_Float> {
 
 // -----
 
+def SPIRV_CLFdimOp : SPIRV_CLBinaryArithmeticOp<"fdim", 24, SPIRV_Float> {
+  let summary = "Compute the positive difference between x and y.";
+
+  let description = [{
+    Returns x - y if x > y, otherwise it returns +0.0.
+
+    Result Type, x and y must be floating-point or vector(2,3,4,8,16) of
+    floating-point values.
+
+    All of the operands, including the Result Type operand, must be of the
+    same type.
+
+    #### Example:
+
+    ```mlir
+    %2 = spirv.CL.fdim %0, %1 : f32
+    %3 = spirv.CL.fdim %0, %1 : vector<3xf16>
+    ```
+  }];
+}
+
+// -----
+
+def SPIRV_CLFmodOp : SPIRV_CLBinaryArithmeticOp<"fmod", 29, SPIRV_Float> {
+  let summary = [{
+    Modulus. Returns x - y * trunc(x / y), with the same sign as x.
+  }];
+
+  let description = [{
+    Result Type, x and y must be floating-point or vector(2,3,4,8,16) of
+    floating-point values.
+
+    All of the operands, including the Result Type operand, must be of the
+    same type.
+
+    #### Example:
+
+    ```mlir
+    %2 = spirv.CL.fmod %0, %1 : f32
+    %3 = spirv.CL.fmod %0, %1 : vector<3xf16>
+    ```
+  }];
+}
+
+// -----
+
+def SPIRV_CLHypotOp : SPIRV_CLBinaryArithmeticOp<"hypot", 32, SPIRV_Float> {
+  let summary = "Compute the square root of x^2 + y^2.";
+
+  let description = [{
+    Result Type, x and y must be floating-point or vector(2,3,4,8,16) of
+    floating-point values.
+
+    All of the operands, including the Result Type operand, must be of the
+    same type.
+
+    #### Example:
+
+    ```mlir
+    %2 = spirv.CL.hypot %0, %1 : f32
+    %3 = spirv.CL.hypot %0, %1 : vector<3xf16>
+    ```
+  }];
+}
+
+// -----
+
 def SPIRV_CLFloorOp : SPIRV_CLUnaryArithmeticOp<"floor", 25, SPIRV_Float> {
   let summary = [{
     Round x to the integral value using the round to negative infinity
diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVGLOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVGLOps.td
index dca21b6eedba0..e0b85e73cad99 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVGLOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVGLOps.td
@@ -1081,6 +1081,83 @@ def SPIRV_GLSClampOp : SPIRV_GLTernaryArithmeticOp<"SClamp", 45, SPIRV_Integer>
 
 // -----
 
+def SPIRV_GLNMaxOp : SPIRV_GLBinaryArithmeticOp<"NMax", 80, SPIRV_Float, [AlwaysSpeculatable]> {
+  let summary = "Return maximum of two floating-point operands, NaN-aware";
+
+  let description = [{
+    Result is y if x < y; otherwise result is x, where x and y are interpreted
+    as floating-point. NMax considers a NaN argument as missing, so if one
+    operand is a NaN the result is the other operand. If both operands are NaNs,
+    the result is a NaN.
+
+    The operands must all be a scalar or vector whose component type is
+    floating-point.
+
+    Result Type and the type of all operands must be the same type. Results are
+    computed per component.
+
+    #### Example:
+
+    ```mlir
+    %2 = spirv.GL.NMax %0, %1 : f32
+    %3 = spirv.GL.NMax %0, %1 : vector<3xf16>
+    ```
+  }];
+}
+
+// -----
+
+def SPIRV_GLNMinOp : SPIRV_GLBinaryArithmeticOp<"NMin", 79, SPIRV_Float, [AlwaysSpeculatable]> {
+  let summary = "Return minimum of two floating-point operands, NaN-aware";
+
+  let description = [{
+    Result is y if y < x; otherwise result is x, where x and y are interpreted
+    as floating-point. NMin considers a NaN argument as missing, so if one
+    operand is a NaN the result is the other operand. If both operands are NaNs,
+    the result is a NaN.
+
+    The operands must all be a scalar or vector whose component type is
+    floating-point.
+
+    Result Type and the type of all operands must be the same type. Results are
+    computed per component.
+
+    #### Example:
+
+    ```mlir
+    %2 = spirv.GL.NMin %0, %1 : f32
+    %3 = spirv.GL.NMin %0, %1 : vector<3xf16>
+    ```
+  }];
+}
+
+// -----
+
+def SPIRV_GLNClampOp : SPIRV_GLTernaryArithmeticOp<"NClamp", 81, SPIRV_Float> {
+  let summary = "Clamp x between min and max values, NaN-aware";
+
+  let description = [{
+    Result is min(max(x, minVal), maxVal). The resulting value is poison if
+    minVal > maxVal. The semantics used by min() and max() are those of NMin
+    and NMax, so NaN arguments are considered missing.
+
+    The operands must all be a scalar or vector whose component type is
+    floating-point.
+
+    Result Type and the type of all operands must be the same type. Results are
+    computed per component.
+
+    #### Example:
+
+    ```mlir
+    %2 = spirv.GL.NClamp %x, %min, %max : f32
+    %3 = spirv.GL.NClamp %x, %min, %max : vector<3xf16>
+    ```
+  }];
+}
+
+// -----
+
 def SPIRV_GLFmaOp : SPIRV_GLTernaryArithmeticOp<"Fma", 50, SPIRV_Float, [AlwaysSpeculatable]> {
   let summary = "Computes a * b + c.";
 
diff --git a/mlir/include/mlir/Dialect/WasmSSA/IR/WasmSSAOps.td b/mlir/include/mlir/Dialect/WasmSSA/IR/WasmSSAOps.td
index a18731f4fd285..bfa0953f50ac8 100644
--- a/mlir/include/mlir/Dialect/WasmSSA/IR/WasmSSAOps.td
+++ b/mlir/include/mlir/Dialect/WasmSSA/IR/WasmSSAOps.td
@@ -318,7 +318,7 @@ def WasmSSA_GlobalOp : WasmSSA_Op<"global", [
     }
     ```
   }];
-  let regions = (region AnyRegion: $initializer);
+  let regions = (region SizedRegion<1>: $initializer);
 
   let extraClassDeclaration = [{
     ::mlir::SymbolTable::Visibility getVisibility() {
@@ -326,8 +326,11 @@ def WasmSSA_GlobalOp : WasmSSA_Op<"global", [
         ::mlir::SymbolTable::Visibility::Public :
         ::mlir::SymbolTable::Visibility::Nested;
     };
+
+    wasmssa::ReturnOp getInitTerminator();
   }];
   let hasCustomAssemblyFormat = 1;
+  let hasVerifier = 1;
 }
 
 def WasmSSA_GlobalImportOp : WasmSSA_Op<"import_global", [
diff --git a/mlir/lib/Conversion/CMakeLists.txt b/mlir/lib/Conversion/CMakeLists.txt
index b43ed905505fa..4fa5b9d582a4e 100644
--- a/mlir/lib/Conversion/CMakeLists.txt
+++ b/mlir/lib/Conversion/CMakeLists.txt
@@ -55,6 +55,7 @@ add_subdirectory(OpenACCToSCF)
 add_subdirectory(OpenMPToLLVM)
 add_subdirectory(PDLToPDLInterp)
 add_subdirectory(PtrToLLVM)
+add_subdirectory(RaiseWasm)
 add_subdirectory(ReconcileUnrealizedCasts)
 add_subdirectory(SCFToControlFlow)
 add_subdirectory(SCFToEmitC)
diff --git a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
index 82d87ede1e4be..61dddaf5155d3 100644
--- a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
+++ b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
@@ -647,6 +647,7 @@ void populateMathToSPIRVPatterns(const SPIRVTypeConverter &typeConverter,
            CheckedElementwiseOpPattern<math::AbsFOp, spirv::GLFAbsOp>,
            CheckedElementwiseOpPattern<math::AbsIOp, spirv::GLSAbsOp>,
            CheckedElementwiseOpPattern<math::AtanOp, spirv::GLAtanOp>,
+           CheckedElementwiseOpPattern<math::Atan2Op, spirv::GLAtan2Op>,
            CheckedElementwiseOpPattern<math::CeilOp, spirv::GLCeilOp>,
            CheckedElementwiseOpPattern<math::ClampFOp, spirv::GLFClampOp>,
            CheckedElementwiseOpPattern<math::CosOp, spirv::GLCosOp>,
diff --git a/mlir/lib/Conversion/RaiseWasm/CMakeLists.txt b/mlir/lib/Conversion/RaiseWasm/CMakeLists.txt
new file mode 100644
index 0000000000000..43b5fd79e49df
--- /dev/null
+++ b/mlir/lib/Conversion/RaiseWasm/CMakeLists.txt
@@ -0,0 +1,19 @@
+add_mlir_conversion_library(MLIRWasmRaise
+  RaiseWasmMLIR.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/RaiseWasm
+
+  DEPENDS
+  MLIRConversionPassIncGen
+
+  LINK_LIBS PUBLIC
+  MLIRArithDialect
+  MLIRControlFlowDialect
+  MLIRFuncDialect
+  MLIRMathDialect
+  MLIRMemRefDialect
+  MLIRTransforms
+  MLIRVectorDialect
+  MLIRWasmSSADialect
+  )
diff --git a/mlir/lib/Conversion/RaiseWasm/RaiseWasmMLIR.cpp b/mlir/lib/Conversion/RaiseWasm/RaiseWasmMLIR.cpp
new file mode 100644
index 0000000000000..83bfde7032ef8
--- /dev/null
+++ b/mlir/lib/Conversion/RaiseWasm/RaiseWasmMLIR.cpp
@@ -0,0 +1,469 @@
+//===- RaiseWasmMLIR.cpp - Convert Wasm to less abstract dialects ---*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements lowering of wasm operations to standard dialects ops.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Conversion/RaiseWasm/RaiseWasmMLIR.h"
+
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/Math/IR/Math.h"
+#include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/Dialect/WasmSSA/IR/WasmSSA.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/BuiltinDialect.h"
+#include "mlir/IR/ValueRange.h"
+#include "mlir/Transforms/DialectConversion.h"
+#include "mlir/Transforms/Passes.h"
+#include "llvm/Support/LogicalResult.h"
+#include <optional>
+
+#define DEBUG_TYPE "wasm-convert"
+
+namespace mlir {
+#define GEN_PASS_DEF_RAISEWASMMLIR
+#include "mlir/Conversion/Passes.h.inc"
+} // namespace mlir
+
+using namespace mlir;
+using namespace mlir::wasmssa;
+namespace {
+
+template <typename SourceOp, typename TargetIntOp, typename TargetFPOp>
+struct IntFPDispatchMappingConversion : OpConversionPattern<SourceOp> {
+  using OpConversionPattern<SourceOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(SourceOp srcOp, typename SourceOp::Adaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    Type type = srcOp.getRhs().getType();
+    if (type.isInteger()) {
+      rewriter.replaceOpWithNewOp<TargetIntOp>(srcOp, srcOp->getResultTypes(),
+                                               adaptor.getOperands());
+      return success();
+    }
+    if (!type.isFloat())
+      return failure();
+    rewriter.replaceOpWithNewOp<TargetFPOp>(srcOp, srcOp->getResultTypes(),
+                                            adaptor.getOperands());
+    return success();
+  }
+};
+
+using WasmAddOpConversion =
+    IntFPDispatchMappingConversion<AddOp, arith::AddIOp, arith::AddFOp>;
+using WasmMulOpConversion =
+    IntFPDispatchMappingConversion<MulOp, arith::MulIOp, arith::MulFOp>;
+using WasmSubOpConversion =
+    IntFPDispatchMappingConversion<SubOp, arith::SubIOp, arith::SubFOp>;
+
+/// Convert a k-ary source operation \p SourceOp into an operation \p TargetOp.
+/// Both \p SourceOp and \p TargetOp must have the same number of operands.
+template <typename SourceOp, typename TargetOp>
+struct OpMappingConversion : OpConversionPattern<SourceOp> {
+  using OpConversionPattern<SourceOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(SourceOp srcOp, typename SourceOp::Adaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    rewriter.replaceOpWithNewOp<TargetOp>(srcOp, srcOp->getResultTypes(),
+                                          adaptor.getOperands());
+    return success();
+  }
+};
+
+using WasmAndOpConversion = OpMappingConversion<AndOp, arith::AndIOp>;
+using WasmCeilOpConversion = OpMappingConversion<CeilOp, math::CeilOp>;
+/// TODO: SIToFP and UIToFP don't allow specification of the floating point
+/// rounding mode
+using WasmConvertSOpConversion =
+    OpMappingConversion<ConvertSOp, arith::SIToFPOp>;
+using WasmConvertUOpConversion =
+    OpMappingConversion<ConvertUOp, arith::UIToFPOp>;
+using WasmDemoteOpConversion = OpMappingConversion<DemoteOp, arith::TruncFOp>;
+using WasmDivFPOpConversion = OpMappingConversion<DivOp, arith::DivFOp>;
+using WasmDivSIOpConversion = OpMappingConversion<DivSIOp, arith::DivSIOp>;
+using WasmDivUIOpConversion = OpMappingConversion<DivUIOp, arith::DivUIOp>;
+using WasmExtendSOpConversion =
+    OpMappingConversion<ExtendSI32Op, arith::ExtSIOp>;
+using WasmExtendUOpConversion =
+    OpMappingConversion<ExtendUI32Op, arith::ExtUIOp>;
+using WasmFloorOpConversion = OpMappingConversion<FloorOp, math::FloorOp>;
+using WasmMaxOpConversion = OpMappingConversion<MaxOp, arith::MaximumFOp>;
+using WasmMinOpConversion = OpMappingConversion<MinOp, arith::MinimumFOp>;
+using WasmOrOpConversion = OpMappingConversion<OrOp, arith::OrIOp>;
+using WasmPromoteOpConversion = OpMappingConversion<PromoteOp, arith::ExtFOp>;
+using WasmRemSIOpConversion = OpMappingConversion<RemSIOp, arith::RemSIOp>;
+using WasmRemUIOpConversion = OpMappingConversion<RemUIOp, arith::RemUIOp>;
+using WasmReinterpretOpConversion =
+    OpMappingConversion<ReinterpretOp, arith::BitcastOp>;
+using WasmShLOpConversion = OpMappingConversion<ShLOp, arith::ShLIOp>;
+using WasmShRSOpConversion = OpMappingConversion<ShRSOp, arith::ShRSIOp>;
+using WasmShRUOpConversion = OpMappingConversion<ShRUOp, arith::ShRUIOp>;
+using WasmXOrOpConversion = OpMappingConversion<XOrOp, arith::XOrIOp>;
+using WasmNegOpConversion = OpMappingConversion<NegOp, arith::NegFOp>;
+using WasmCopySignOpConversion =
+    OpMappingConversion<CopySignOp, math::CopySignOp>;
+using WasmClzOpConversion =
+    OpMappingConversion<ClzOp, math::CountLeadingZerosOp>;
+using WasmCtzOpConversion =
+    OpMappingConversion<CtzOp, math::CountTrailingZerosOp>;
+using WasmPopCntOpConversion = OpMappingConversion<PopCntOp, math::CtPopOp>;
+using WasmAbsOpConversion = OpMappingConversion<AbsOp, math::AbsFOp>;
+using WasmTruncOpConversion = OpMappingConversion<TruncOp, math::TruncOp>;
+using WasmSqrtOpConversion = OpMappingConversion<SqrtOp, math::SqrtOp>;
+using WasmWrapOpConversion = OpMappingConversion<WrapOp, arith::TruncIOp>;
+
+struct WasmCallOpConversion : OpConversionPattern<FuncCallOp> {
+  using OpConversionPattern::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(FuncCallOp funcCallOp, FuncCallOp::Adaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    rewriter.replaceOpWithNewOp<func::CallOp>(
+        funcCallOp, funcCallOp.getCallee(), funcCallOp.getResults().getTypes(),
+        funcCallOp.getOperands());
+    return success();
+  }
+};
+
+struct WasmConstOpConversion : OpConversionPattern<ConstOp> {
+  using OpConversionPattern::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(ConstOp constOp, ConstOp::Adaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    rewriter.replaceOpWithNewOp<arith::ConstantOp>(constOp, constOp.getValue());
+    return success();
+  }
+};
+
+struct WasmFuncImportOpConversion : OpConversionPattern<FuncImportOp> {
+  using OpConversionPattern::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(FuncImportOp funcImportOp, FuncImportOp::Adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto nFunc = rewriter.replaceOpWithNewOp<func::FuncOp>(
+        funcImportOp, funcImportOp.getSymName(), funcImportOp.getType());
+    nFunc.setVisibility(SymbolTable::Visibility::Private);
+    return success();
+  }
+};
+
+struct WasmFuncOpConversion : OpConversionPattern<FuncOp> {
+  using OpConversionPattern::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(FuncOp funcOp, FuncOp::Adaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto newFunc =
+        func::FuncOp::create(rewriter, funcOp->getLoc(), funcOp.getSymName(),
+                             funcOp.getFunctionType());
+    rewriter.cloneRegionBefore(funcOp.getBody(), newFunc.getBody(),
+                               newFunc.getBody().end());
+    Block *oldEntryBlock = &newFunc.getBody().front();
+    auto blockArgTypes = oldEntryBlock->getArgumentTypes();
+    TypeConverter::SignatureConversion sC{oldEntryBlock->getNumArguments()};
+    auto numArgs = blockArgTypes.size();
+    for (size_t i = 0; i < numArgs; ++i) {
+      auto argType = dyn_cast<LocalRefType>(blockArgTypes[i]);
+      if (!argType)
+        return failure();
+      sC.addInputs(i, argType.getElementType());
+    }
+
+    rewriter.applySignatureConversion(oldEntryBlock, sC, getTypeConverter());
+    rewriter.replaceOp(funcOp, newFunc);
+    return success();
+  }
+};
+
+struct WasmGlobalImportOpConverter : OpConversionPattern<GlobalImportOp> {
+  using OpConversionPattern::OpConversionPattern;
+  LogicalResult
+  matchAndRewrite(GlobalImportOp gIOp, GlobalImportOp::Adaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto memrefGOp = rewriter.replaceOpWithNewOp<memref::GlobalOp>(
+        gIOp, gIOp.getSymNameAttr(), rewriter.getStringAttr("nested"),
+        TypeAttr::get(MemRefType::get({1}, gIOp.getType())), Attribute{},
+        /*constant*/ UnitAttr{},
+        /*alignment*/ IntegerAttr{});
+    memrefGOp.setConstant(!gIOp.getIsMutable());
+    return success();
+  }
+};
+
+template <typename CRTP, typename OriginOpType>
+struct GlobalOpConverter : OpConversionPattern<GlobalOp> {
+  using OpConversionPattern::OpConversionPattern;
+  LogicalResult
+  matchAndRewrite(GlobalOp globalOp, GlobalOp::Adaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    ReturnOp rop = globalOp.getInitTerminator();
+
+    if (rop->getNumOperands() != 1)
+      return rewriter.notifyMatchFailure(
+          globalOp, "globalOp initializer should return one value exactly");
+
+    auto initializerOp =
+        dyn_cast<OriginOpType>(rop->getOperand(0).getDefiningOp());
+
+    if (!initializerOp)
+      return rewriter.notifyMatchFailure(
+          globalOp, "invalid initializer op type for this pattern");
+
+    return static_cast<CRTP const *>(this)->handleInitializer(
+        globalOp, rewriter, initializerOp);
+  }
+};
+
+struct WasmGlobalWithConstInitConversion
+    : GlobalOpConverter<WasmGlobalWithConstInitConversion, ConstOp> {
+  using GlobalOpConverter::GlobalOpConverter;
+  LogicalResult handleInitializer(GlobalOp globalOp,
+                                  ConversionPatternRewriter &rewriter,
+                                  ConstOp constInit) const {
+    auto initializer =
+        DenseElementsAttr::get(RankedTensorType::get({1}, globalOp.getType()),
+                               ArrayRef<Attribute>{constInit.getValueAttr()});
+    auto globalReplacement = rewriter.replaceOpWithNewOp<memref::GlobalOp>(
+        globalOp, globalOp.getSymNameAttr(), rewriter.getStringAttr("private"),
+        TypeAttr::get(MemRefType::get({1}, globalOp.getType())), initializer,
+        /*constant*/ UnitAttr{},
+        /*alignment*/ IntegerAttr{});
+    globalReplacement.setConstant(!globalOp.getIsMutable());
+    return success();
+  }
+};
+
+struct WasmGlobalWithGetGlobalInitConversion
+    : GlobalOpConverter<WasmGlobalWithGetGlobalInitConversion, GlobalGetOp> {
+  using GlobalOpConverter::GlobalOpConverter;
+  LogicalResult handleInitializer(GlobalOp globalOp,
+                                  ConversionPatternRewriter &rewriter,
+                                  GlobalGetOp constInit) const {
+    auto globalReplacement = rewriter.replaceOpWithNewOp<memref::GlobalOp>(
+        globalOp, globalOp.getSymNameAttr(), rewriter.getStringAttr("private"),
+        TypeAttr::get(MemRefType::get({1}, globalOp.getType())),
+        rewriter.getUnitAttr(),
+        /*constant*/ UnitAttr{},
+        /*alignment*/ IntegerAttr{});
+    globalReplacement.setConstant(!globalOp.getIsMutable());
+    auto loc = globalOp.getLoc();
+    auto initializerName = (globalOp.getSymName() + "::initializer").str();
+    auto globalInitializer =
+        func::FuncOp::create(rewriter, loc, initializerName,
+                             FunctionType::get(getContext(), {}, {}));
+    globalInitializer->setAttr(rewriter.getStringAttr("initializer"),
+                               rewriter.getUnitAttr());
+    auto *initializerBody = globalInitializer.addEntryBlock();
+    auto sip = rewriter.saveInsertionPoint();
+    rewriter.setInsertionPointToStart(initializerBody);
+    auto srcGlobalPtr = memref::GetGlobalOp::create(
+        rewriter, loc, MemRefType::get({1}, constInit.getType()),
+        constInit.getGlobal());
+    auto destGlobalPtr =
+        memref::GetGlobalOp::create(rewriter, loc, globalReplacement.getType(),
+                                    globalReplacement.getSymName());
+    auto idx = arith::ConstantIndexOp::create(rewriter, loc, 0).getResult();
+    auto loadSrc =
+        memref::LoadOp::create(rewriter, loc, srcGlobalPtr, ValueRange{idx});
+    memref::StoreOp::create(rewriter, loc, loadSrc.getResult(),
+                            destGlobalPtr.getResult(), ValueRange{idx});
+    func::ReturnOp::create(rewriter, loc);
+    rewriter.restoreInsertionPoint(sip);
+    return success();
+  }
+};
+
+inline TypedAttr getInitializerAttr(Type t) {
+  assert(t.isIntOrFloat() &&
+         "This helper is intended to use with int and float types");
+  if (t.isInteger())
+    return IntegerAttr::get(t, 0);
+  if (t.isFloat())
+    return FloatAttr::get(t, 0.);
+  return TypedAttr{};
+}
+
+struct WasmLocalConversion : OpConversionPattern<LocalOp> {
+  using OpConversionPattern::OpConversionPattern;
+  LogicalResult
+  matchAndRewrite(LocalOp localOp, LocalOp::Adaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto alloca = rewriter.replaceOpWithNewOp<memref::AllocaOp>(
+        localOp,
+        MemRefType::get({}, localOp.getResult().getType().getElementType()));
+    auto initializer = arith::ConstantOp::create(
+        rewriter, localOp->getLoc(),
+        getInitializerAttr(localOp.getResult().getType().getElementType()));
+    memref::StoreOp::create(rewriter, localOp->getLoc(),
+                            initializer.getResult(), alloca.getResult());
+    return success();
+  }
+};
+
+struct WasmLocalGetConversion : OpConversionPattern<LocalGetOp> {
+  using OpConversionPattern::OpConversionPattern;
+  LogicalResult
+  matchAndRewrite(LocalGetOp localGetOp, LocalGetOp::Adaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    rewriter.replaceOpWithNewOp<memref::LoadOp>(
+        localGetOp, localGetOp.getResult().getType(), adaptor.getLocalVar(),
+        ValueRange{});
+    return success();
+  }
+};
+
+struct WasmLocalSetConversion : OpConversionPattern<LocalSetOp> {
+  using OpConversionPattern::OpConversionPattern;
+  LogicalResult
+  matchAndRewrite(LocalSetOp localSetOp, LocalSetOp::Adaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    rewriter.replaceOpWithNewOp<memref::StoreOp>(
+        localSetOp, adaptor.getValue(), adaptor.getLocalVar(), ValueRange{});
+    return success();
+  }
+};
+
+struct WasmLocalTeeConversion : OpConversionPattern<LocalTeeOp> {
+  using OpConversionPattern::OpConversionPattern;
+  LogicalResult
+  matchAndRewrite(LocalTeeOp localTeeOp, LocalTeeOp::Adaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    memref::StoreOp::create(rewriter, localTeeOp->getLoc(), adaptor.getValue(),
+                            adaptor.getLocalVar());
+    rewriter.replaceOp(localTeeOp, adaptor.getValue());
+    return success();
+  }
+};
+
+struct WasmReturnOpConversion : OpConversionPattern<ReturnOp> {
+  using OpConversionPattern::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(ReturnOp returnOp, ReturnOp::Adaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    rewriter.replaceOpWithNewOp<func::ReturnOp>(returnOp,
+                                                adaptor.getOperands());
+    return success();
+  }
+};
+
+struct RaiseWasmMLIRPass : public impl::RaiseWasmMLIRBase<RaiseWasmMLIRPass> {
+  void runOnOperation() override {
+    ConversionTarget target{getContext()};
+    target.addIllegalDialect<WasmSSADialect>();
+    target.addLegalDialect<arith::ArithDialect, BuiltinDialect,
+                           cf::ControlFlowDialect, func::FuncDialect,
+                           memref::MemRefDialect, math::MathDialect>();
+    RewritePatternSet patterns(&getContext());
+    TypeConverter tc{};
+    tc.addConversion([](Type type) -> std::optional<Type> { return type; });
+    tc.addConversion([](LocalRefType type) -> std::optional<Type> {
+      return MemRefType::get({}, type.getElementType());
+    });
+    tc.addTargetMaterialization([](OpBuilder &builder, MemRefType destType,
+                                   ValueRange values, Location loc) -> Value {
+      if (values.size() != 1 ||
+          values.front().getType() != destType.getElementType())
+        return {};
+      auto localVar = memref::AllocaOp::create(builder, loc, destType);
+      memref::StoreOp::create(builder, loc, values.front(),
+                              localVar.getResult());
+      return localVar.getResult();
+    });
+    populateRaiseWasmMLIRConversionPatterns(tc, patterns);
+
+    llvm::DenseMap<StringAttr, StringAttr> idxSymToImportSym{};
+    auto *topOp = getOperation();
+    topOp->walk([&idxSymToImportSym, this](ImportOpInterface importOp) {
+      auto const qualifiedImportName = importOp.getQualifiedImportName();
+      auto qualNameAttr = StringAttr::get(&getContext(), qualifiedImportName);
+      idxSymToImportSym.insert(
+          std::make_pair(importOp.getSymbolName(), qualNameAttr));
+    });
+
+    if (failed(applyFullConversion(topOp, target, std::move(patterns))))
+      return signalPassFailure();
+
+    auto symTable = SymbolTable{topOp};
+    for (auto &[oldName, newName] : idxSymToImportSym) {
+      if (failed(symTable.rename(oldName, newName)))
+        return signalPassFailure();
+    }
+  }
+};
+} // namespace
+
+void mlir::populateRaiseWasmMLIRConversionPatterns(
+    TypeConverter &tc, RewritePatternSet &patternSet) {
+  auto *ctx = patternSet.getContext();
+  // Disable clang-format in patternSet for readability + small diffs.
+  // clang-format off
+  patternSet
+      .add<
+           WasmAbsOpConversion,
+           WasmAddOpConversion,
+           WasmAndOpConversion,
+           WasmCallOpConversion,
+           WasmCeilOpConversion,
+           WasmClzOpConversion,
+           WasmConstOpConversion,
+           WasmConvertSOpConversion,
+           WasmConvertUOpConversion,
+           WasmCopySignOpConversion,
+           WasmCtzOpConversion,
+           WasmDemoteOpConversion,
+           WasmDivFPOpConversion,
+           WasmDivSIOpConversion,
+           WasmDivUIOpConversion,
+           WasmExtendSOpConversion,
+           WasmExtendUOpConversion,
+           WasmFloorOpConversion,
+           WasmFuncImportOpConversion,
+           WasmFuncOpConversion,
+           WasmGlobalImportOpConverter,
+           WasmGlobalWithConstInitConversion,
+           WasmGlobalWithGetGlobalInitConversion,
+           WasmLocalConversion,
+           WasmLocalGetConversion,
+           WasmLocalSetConversion,
+           WasmLocalTeeConversion,
+           WasmMaxOpConversion,
+           WasmMinOpConversion,
+           WasmMulOpConversion,
+           WasmNegOpConversion,
+           WasmOrOpConversion,
+           WasmPopCntOpConversion,
+           WasmPromoteOpConversion,
+           WasmReinterpretOpConversion,
+           WasmRemSIOpConversion,
+           WasmRemUIOpConversion,
+           WasmReturnOpConversion,
+           WasmShLOpConversion,
+           WasmShRSOpConversion,
+           WasmShRUOpConversion,
+           WasmSqrtOpConversion,
+           WasmSubOpConversion,
+           WasmTruncOpConversion,
+           WasmWrapOpConversion,
+           WasmXOrOpConversion
+           >(tc, ctx);
+  // clang-format on
+}
+
+std::unique_ptr<Pass> createRaiseWasmMLIRPass() {
+  return std::make_unique<RaiseWasmMLIRPass>();
+}
diff --git a/mlir/lib/Dialect/OpenACC/Transforms/ACCEmitRemarksPrivate.cpp b/mlir/lib/Dialect/OpenACC/Transforms/ACCEmitRemarksPrivate.cpp
new file mode 100644
index 0000000000000..645080aea0b79
--- /dev/null
+++ b/mlir/lib/Dialect/OpenACC/Transforms/ACCEmitRemarksPrivate.cpp
@@ -0,0 +1,100 @@
+//===- ACCEmitRemarksPrivate.cpp - Emit OpenACC privatization remarks ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass emits remarks describing the private and firstprivate variables
+// associated with OpenACC compute and loop constructs.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h"
+#include "mlir/Dialect/OpenACC/OpenACC.h"
+#include "mlir/Dialect/OpenACC/Transforms/Passes.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/ADT/TypeSwitch.h"
+
+namespace mlir {
+namespace acc {
+#define GEN_PASS_DEF_ACCEMITREMARKSPRIVATE
+#include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc"
+} // namespace acc
+} // namespace mlir
+
+#define DEBUG_TYPE "acc-emit-remarks-private"
+
+using namespace mlir;
+
+namespace {
+
+template <typename OpTy>
+static void reportPrivatization(Operation *accOp, ValueRange operands,
+                                acc::OpenACCSupport &accSupport,
+                                StringRef clause) {
+  SmallVector<std::string> implicitNames;
+  SmallVector<std::string> explicitNames;
+  for (Value operand : operands) {
+    auto op = cast<OpTy>(operand.getDefiningOp());
+    std::string varName = accSupport.getVariableName(op.getAccVar());
+    (op.getImplicit() ? implicitNames : explicitNames)
+        .push_back(varName.empty() ? "<unknown>" : varName);
+  }
+
+  if (!implicitNames.empty())
+    accSupport.emitRemark(
+        accOp,
+        [clause, names = std::move(implicitNames)]() {
+          return (Twine("Generating implicit ") + clause + "(" +
+                  llvm::join(names, ",") + ")")
+              .str();
+        },
+        DEBUG_TYPE);
+
+  if (!explicitNames.empty())
+    accSupport.emitRemark(
+        accOp,
+        [clause, names = std::move(explicitNames)]() {
+          return (Twine("Generating ") + clause + "(" + llvm::join(names, ",") +
+                  ")")
+              .str();
+        },
+        DEBUG_TYPE);
+}
+
+template <typename OpTy>
+static void emitRemarksForACCOp(OpTy accOp, acc::OpenACCSupport &accSupport) {
+  reportPrivatization<acc::FirstprivateOp>(
+      accOp, accOp.getFirstprivateOperands(), accSupport, "firstprivate");
+  reportPrivatization<acc::PrivateOp>(accOp, accOp.getPrivateOperands(),
+                                      accSupport, "private");
+}
+
+class ACCEmitRemarksPrivate
+    : public acc::impl::ACCEmitRemarksPrivateBase<ACCEmitRemarksPrivate> {
+public:
+  using ACCEmitRemarksPrivateBase<
+      ACCEmitRemarksPrivate>::ACCEmitRemarksPrivateBase;
+
+  void runOnOperation() override {
+    func::FuncOp func = getOperation();
+
+    auto cachedAnalysis = getCachedParentAnalysis<acc::OpenACCSupport>();
+    acc::OpenACCSupport &accSupport = cachedAnalysis
+                                          ? cachedAnalysis->get()
+                                          : getAnalysis<acc::OpenACCSupport>();
+
+    func.walk([&](Operation *op) {
+      TypeSwitch<Operation *>(op).Case<ACC_COMPUTE_CONSTRUCT_AND_LOOP_OPS>(
+          [&](auto constructOp) {
+            emitRemarksForACCOp(constructOp, accSupport);
+          });
+    });
+  }
+};
+
+} // namespace
diff --git a/mlir/lib/Dialect/OpenACC/Transforms/ACCSpecializeForHost.cpp b/mlir/lib/Dialect/OpenACC/Transforms/ACCSpecializeForHost.cpp
index 633538069c268..d3c8b683b76cb 100644
--- a/mlir/lib/Dialect/OpenACC/Transforms/ACCSpecializeForHost.cpp
+++ b/mlir/lib/Dialect/OpenACC/Transforms/ACCSpecializeForHost.cpp
@@ -344,6 +344,9 @@ class ACCSpecializeForHost
         populateACCOrphanToHostPatterns(patterns, accSupport);
       GreedyRewriteConfig config;
       config.setUseTopDownTraversal(true);
+      // Deeply nested orphan acc.loops can need more than the default
+      // iteration cap to converge; lift it to avoid spurious pass failure.
+      config.setMaxIterations(GreedyRewriteConfig::kNoLimit);
       if (failed(applyPatternsGreedily(funcOp, std::move(patterns), config)))
         signalPassFailure();
     }
@@ -461,8 +464,8 @@ void mlir::acc::populateACCHostFallbackPatterns(RewritePatternSet &patterns,
   // Runtime operations - erase them
   patterns.insert<
       ACCOpEraseConversion<acc::InitOp>, ACCOpEraseConversion<acc::ShutdownOp>,
-      ACCOpEraseConversion<acc::SetOp>, ACCOpEraseConversion<acc::WaitOp>,
-      ACCOpEraseConversion<acc::TerminatorOp>>(context);
+      ACCOpEraseConversion<acc::SetOp>, ACCOpEraseConversion<acc::WaitOp>>(
+      context);
 
   // Compute constructs - unwrap their regions
   patterns.insert<ACCRegionUnwrapConversion<acc::ParallelOp>,
diff --git a/mlir/lib/Dialect/OpenACC/Transforms/CMakeLists.txt b/mlir/lib/Dialect/OpenACC/Transforms/CMakeLists.txt
index 2ae3571673469..d977028c283e6 100644
--- a/mlir/lib/Dialect/OpenACC/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/OpenACC/Transforms/CMakeLists.txt
@@ -5,6 +5,7 @@ add_mlir_dialect_library(MLIROpenACCTransforms
   ACCRoutineToGPUFunc.cpp
   ACCDeclareGPUModuleInsertion.cpp
   ACCEmitRemarksLoop.cpp
+  ACCEmitRemarksPrivate.cpp
   ACCIfClauseLowering.cpp
   ACCImplicitData.cpp
   ACCRecipeMaterialization.cpp
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index ef8c1f74130bb..1ce9fb38de746 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -3927,6 +3927,24 @@ mlir::omp ::decodeCli(Value cli) {
   return {create, gen, cons};
 }
 
+ClauseProcBindKind
+mlir::omp::convertProcBindKind(llvm::omp::ProcBindKind kind) {
+  switch (kind) {
+  case llvm::omp::ProcBindKind::OMP_PROC_BIND_close:
+    return ClauseProcBindKind::Close;
+  case llvm::omp::ProcBindKind::OMP_PROC_BIND_master:
+    return ClauseProcBindKind::Master;
+  case llvm::omp::ProcBindKind::OMP_PROC_BIND_primary:
+    return ClauseProcBindKind::Primary;
+  case llvm::omp::ProcBindKind::OMP_PROC_BIND_spread:
+    return ClauseProcBindKind::Spread;
+  case llvm::omp::ProcBindKind::OMP_PROC_BIND_default:
+  case llvm::omp::ProcBindKind::OMP_PROC_BIND_unknown:
+    break;
+  }
+  llvm_unreachable("unexpected proc-bind kind");
+}
+
 void NewCliOp::build(::mlir::OpBuilder &odsBuilder,
                      ::mlir::OperationState &odsState) {
   odsState.addTypes(CanonicalLoopInfoType::get(odsBuilder.getContext()));
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
index 4cb6e6bbf6579..f699055b5502f 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
@@ -1284,6 +1284,16 @@ ParseResult spirv::GLSClampOp::parse(OpAsmParser &parser,
 }
 void spirv::GLSClampOp::print(OpAsmPrinter &p) { printOneResultOp(*this, p); }
 
+//===----------------------------------------------------------------------===//
+// spirv.GLNClampOp
+//===----------------------------------------------------------------------===//
+
+ParseResult spirv::GLNClampOp::parse(OpAsmParser &parser,
+                                     OperationState &result) {
+  return parseOneResultSameOperandTypeOp(parser, result);
+}
+void spirv::GLNClampOp::print(OpAsmPrinter &p) { printOneResultOp(*this, p); }
+
 //===----------------------------------------------------------------------===//
 // spirv.GLFmaOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/WasmSSA/IR/WasmSSAOps.cpp b/mlir/lib/Dialect/WasmSSA/IR/WasmSSAOps.cpp
index f3eb601bed5c3..08de887a56a97 100644
--- a/mlir/lib/Dialect/WasmSSA/IR/WasmSSAOps.cpp
+++ b/mlir/lib/Dialect/WasmSSA/IR/WasmSSAOps.cpp
@@ -246,6 +246,16 @@ void FuncImportOp::build(OpBuilder &odsBuilder, OperationState &odsState,
 //===----------------------------------------------------------------------===//
 // GlobalOp
 //===----------------------------------------------------------------------===//
+namespace {
+Operation *getGlobalOpTerminatorOp(GlobalOp gop) {
+  return gop.getInitializer().begin()->getTerminator();
+}
+} // namespace
+
+ReturnOp GlobalOp::getInitTerminator() {
+  return llvm::cast<wasmssa::ReturnOp>(getGlobalOpTerminatorOp(*this));
+}
+
 // Custom formats
 ParseResult GlobalOp::parse(OpAsmParser &parser, OperationState &result) {
   StringAttr symbolName;
@@ -292,6 +302,10 @@ void GlobalOp::print(OpAsmPrinter &printer) {
   }
 }
 
+LogicalResult GlobalOp::verify() {
+  return success(llvm::isa<ReturnOp>(getGlobalOpTerminatorOp(*this)));
+}
+
 //===----------------------------------------------------------------------===//
 // GlobalGetOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Conversion/MathToSPIRV/math-to-gl-spirv.mlir b/mlir/test/Conversion/MathToSPIRV/math-to-gl-spirv.mlir
index a8b2975ecb3c7..45e7bb01edc0b 100644
--- a/mlir/test/Conversion/MathToSPIRV/math-to-gl-spirv.mlir
+++ b/mlir/test/Conversion/MathToSPIRV/math-to-gl-spirv.mlir
@@ -139,6 +139,20 @@ func.func @float32_ternary_vector(%a: vector<4xf32>, %b: vector<4xf32>,
   return
 }
 
+// CHECK-LABEL: @float32_binary_scalar
+func.func @float32_binary_scalar(%lhs: f32, %rhs: f32) {
+  // CHECK: spirv.GL.Atan2 %{{.*}}, %{{.*}} : f32
+  %0 = math.atan2 %lhs, %rhs : f32
+  return
+}
+
+// CHECK-LABEL: @float32_binary_vector
+func.func @float32_binary_vector(%lhs: vector<3xf32>, %rhs: vector<3xf32>) {
+  // CHECK: spirv.GL.Atan2 %{{.*}}, %{{.*}} : vector<3xf32>
+  %0 = math.atan2 %lhs, %rhs : vector<3xf32>
+  return
+}
+
 // CHECK-LABEL: @float32_clamp_scalar
 func.func @float32_clamp_scalar(%value: f32, %min: f32, %max: f32) {
   // CHECK: spirv.GL.FClamp %{{.*}}, %{{.*}}, %{{.*}} : f32
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-abs-to-math-abs.mlir b/mlir/test/Conversion/RaiseWasm/wasm-abs-to-math-abs.mlir
new file mode 100644
index 0000000000000..eee03772589b9
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-abs-to-math-abs.mlir
@@ -0,0 +1,27 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir -o - | FileCheck %s
+
+// CHECK-LABEL:   func.func @abs_f32(
+// CHECK-SAME:      %[[ARG0:.*]]: f32) -> f32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<f32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_0]][] : memref<f32>
+// CHECK:           %[[VAL_1:.*]] = memref.load %[[VAL_0]][] : memref<f32>
+// CHECK:           %[[VAL_2:.*]] = math.absf %[[VAL_1]] : f32
+// CHECK:           return %[[VAL_2]] : f32
+wasmssa.func @abs_f32(%arg0: !wasmssa<local ref to f32>) -> f32 {
+    %val = wasmssa.local_get %arg0 : ref to f32
+    %op = wasmssa.abs %val : f32
+    wasmssa.return %op : f32
+}
+
+// CHECK-LABEL:   func.func @abs_f64(
+// CHECK-SAME:      %[[ARG0:.*]]: f64) -> f64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<f64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_0]][] : memref<f64>
+// CHECK:           %[[VAL_1:.*]] = memref.load %[[VAL_0]][] : memref<f64>
+// CHECK:           %[[VAL_2:.*]] = math.absf %[[VAL_1]] : f64
+// CHECK:           return %[[VAL_2]] : f64
+wasmssa.func @abs_f64(%arg0: !wasmssa<local ref to f64>) -> f64 {
+    %val = wasmssa.local_get %arg0 : ref to f64
+    %op = wasmssa.abs %val : f64
+    wasmssa.return %op : f64
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-add-to-arith-add.mlir b/mlir/test/Conversion/RaiseWasm/wasm-add-to-arith-add.mlir
new file mode 100644
index 0000000000000..377be5d1a2f1e
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-add-to-arith-add.mlir
@@ -0,0 +1,79 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir | FileCheck %s
+
+// CHECK-LABEL:   func.func @func_1(
+// CHECK-SAME:                      %[[ARG0:.*]]: i32,
+// CHECK-SAME:                      %[[ARG1:.*]]: i32) -> i32 {
+wasmssa.func @func_1(%arg0: !wasmssa<local ref to i32>, %arg1: !wasmssa<local ref to i32>) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i32>
+%v0 = wasmssa.local_get %arg0 : ref to i32
+%v1 = wasmssa.local_get %arg1 : ref to i32
+// CHECK:           %[[VAL_4:.*]] = arith.addi %[[VAL_2]], %[[VAL_3]] : i32
+%0 = wasmssa.add %v0 %v1 : i32
+// CHECK:           return %[[VAL_4]] : i32
+wasmssa.return %0 : i32
+}
+
+// -----
+
+// CHECK-LABEL:   func.func @func_2(
+// CHECK-SAME:                      %[[ARG0:.*]]: i64,
+// CHECK-SAME:                      %[[ARG1:.*]]: i64) -> i64 {
+wasmssa.func @func_2(%arg0: !wasmssa<local ref to i64>, %arg1: !wasmssa<local ref to i64>) -> i64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i64>
+%v0 = wasmssa.local_get %arg0 : ref to i64
+%v1 = wasmssa.local_get %arg1 : ref to i64
+// CHECK:           %[[VAL_4:.*]] = arith.addi %[[VAL_2]], %[[VAL_3]] : i64
+%0 = wasmssa.add %v0 %v1 : i64
+// CHECK:           return %[[VAL_4]] : i64
+wasmssa.return %0 : i64
+}
+
+// -----
+
+// CHECK-LABEL:   func.func @func_3(
+// CHECK-SAME:                      %[[ARG0:.*]]: f32,
+// CHECK-SAME:                      %[[ARG1:.*]]: f32) -> f32 {
+wasmssa.func @func_3(%arg0: !wasmssa<local ref to f32>, %arg1: !wasmssa<local ref to f32>) -> f32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<f32>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<f32>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<f32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<f32>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<f32>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<f32>
+%v0 = wasmssa.local_get %arg0 : ref to f32
+%v1 = wasmssa.local_get %arg1 : ref to f32
+// CHECK:           %[[VAL_4:.*]] = arith.addf %[[VAL_2]], %[[VAL_3]] : f32
+%0 = wasmssa.add %v0 %v1 : f32
+// CHECK:           return %[[VAL_4]] : f32
+wasmssa.return %0 : f32
+}
+
+// -----
+
+// CHECK-LABEL:   func.func @func_4(
+// CHECK-SAME:                      %[[ARG0:.*]]: f64,
+// CHECK-SAME:                      %[[ARG1:.*]]: f64) -> f64 {
+wasmssa.func @func_4(%arg0: !wasmssa<local ref to f64>, %arg1: !wasmssa<local ref to f64>) -> f64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<f64>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<f64>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<f64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<f64>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<f64>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<f64>
+%v0 = wasmssa.local_get %arg0 : ref to f64
+%v1 = wasmssa.local_get %arg1 : ref to f64
+// CHECK:           %[[VAL_4:.*]] = arith.addf %[[VAL_2]], %[[VAL_3]] : f64
+%0 = wasmssa.add %v0 %v1 : f64
+// CHECK:           return %[[VAL_4]] : f64
+wasmssa.return %0 : f64
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-and-to-arith-and.mlir b/mlir/test/Conversion/RaiseWasm/wasm-and-to-arith-and.mlir
new file mode 100644
index 0000000000000..eebc76e7322ae
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-and-to-arith-and.mlir
@@ -0,0 +1,38 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir -o - | FileCheck %s
+
+
+// CHECK-LABEL:   func.func @and_i32(
+// CHECK-SAME:                      %[[ARG0:.*]]: i32,
+// CHECK-SAME:                      %[[ARG1:.*]]: i32) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_4:.*]] = arith.andi %[[VAL_2]], %[[VAL_3]] : i32
+// CHECK:           return %[[VAL_4]] : i32
+wasmssa.func exported @and_i32(%arg0: !wasmssa<local ref to i32>, %arg1: !wasmssa<local ref to i32>) -> i32 {
+    %v0 = wasmssa.local_get %arg0 : ref to i32
+    %v1 = wasmssa.local_get %arg1 : ref to i32
+    %and = wasmssa.and %v0 %v1 : i32
+    wasmssa.return %and : i32
+}
+
+// CHECK-LABEL:   func.func @and_i64(
+// CHECK-SAME:                      %[[ARG0:.*]]: i64,
+// CHECK-SAME:                      %[[ARG1:.*]]: i64) -> i64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_4:.*]] = arith.andi %[[VAL_2]], %[[VAL_3]] : i64
+// CHECK:           return %[[VAL_4]] : i64
+wasmssa.func exported @and_i64(%arg0: !wasmssa<local ref to i64>, %arg1: !wasmssa<local ref to i64>) -> i64 {
+    %v0 = wasmssa.local_get %arg0 : ref to i64
+    %v1 = wasmssa.local_get %arg1 : ref to i64
+    %and = wasmssa.and %v0 %v1 : i64
+    wasmssa.return %and : i64
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-ceil-to-math-ceil.mlir b/mlir/test/Conversion/RaiseWasm/wasm-ceil-to-math-ceil.mlir
new file mode 100644
index 0000000000000..15875f60c1c43
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-ceil-to-math-ceil.mlir
@@ -0,0 +1,24 @@
+// RUN: mlir-opt %s --raise-wasm-mlir | FileCheck %s
+
+module {
+  wasmssa.func exported @func_0() -> f64 {
+    %0 = wasmssa.const -1.210000e+01 : f64
+    %1 = wasmssa.ceil %0 : f64
+    wasmssa.return %1 : f64
+  }
+  wasmssa.func exported @func_1() -> f32 {
+    %0 = wasmssa.const 1.618000e+00 : f32
+    %1 = wasmssa.ceil %0 : f32
+    wasmssa.return %1 : f32
+  }
+}
+
+// CHECK-LABEL:   func.func @func_0() -> f64 {
+// CHECK:           %[[VAL_0:.*]] = arith.constant -1.210000e+01 : f64
+// CHECK:           %[[VAL_1:.*]] = math.ceil %[[VAL_0]] : f64
+// CHECK:           return %[[VAL_1]] : f64
+
+// CHECK-LABEL:   func.func @func_1() -> f32 {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1.618000e+00 : f32
+// CHECK:           %[[VAL_1:.*]] = math.ceil %[[VAL_0]] : f32
+// CHECK:           return %[[VAL_1]] : f32
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-clz-to-math-clz.mlir b/mlir/test/Conversion/RaiseWasm/wasm-clz-to-math-clz.mlir
new file mode 100644
index 0000000000000..944cf24ff40c2
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-clz-to-math-clz.mlir
@@ -0,0 +1,27 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir -o - | FileCheck %s
+
+// CHECK-LABEL:   func.func @clz_i32(
+// CHECK-SAME:      %[[ARG0:.*]]: i32) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_1:.*]] = memref.load %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_2:.*]] = math.ctlz %[[VAL_1]] : i32
+// CHECK:           return %[[VAL_2]] : i32
+wasmssa.func exported @clz_i32(%arg0: !wasmssa<local ref to i32>) -> i32 {
+    %v0 = wasmssa.local_get %arg0 : ref to i32
+    %op = wasmssa.clz %v0 : i32
+    wasmssa.return %op : i32
+}
+
+// CHECK-LABEL:   func.func @clz_i64(
+// CHECK-SAME:      %[[ARG0:.*]]: i64) -> i64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_1:.*]] = memref.load %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_2:.*]] = math.ctlz %[[VAL_1]] : i64
+// CHECK:           return %[[VAL_2]] : i64
+wasmssa.func exported @clz_i64(%arg0: !wasmssa<local ref to i64>) -> i64 {
+    %v0 = wasmssa.local_get %arg0 : ref to i64
+    %op = wasmssa.clz %v0 : i64
+    wasmssa.return %op : i64
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-const-to-arith-const.mlir b/mlir/test/Conversion/RaiseWasm/wasm-const-to-arith-const.mlir
new file mode 100644
index 0000000000000..06bd0acab4f96
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-const-to-arith-const.mlir
@@ -0,0 +1,15 @@
+// RUN: mlir-opt %s --raise-wasm-mlir | FileCheck %s
+
+// CHECK-LABEL:   func.func @get_some_const() -> (i32, i64, f32, f64) {
+wasmssa.func exported @get_some_const() -> (i32, i64, f32, f64) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 17 : i32
+%0 = wasmssa.const 17: i32
+// CHECK:           %[[VAL_1:.*]] = arith.constant -163 : i64
+%1 = wasmssa.const -163 : i64
+// CHECK:           %[[VAL_2:.*]] = arith.constant 3.140000e+00 : f32
+%2 = wasmssa.const 3.14 : f32
+// CHECK:           %[[VAL_3:.*]] = arith.constant -1.575000e+02 : f64
+%3 = wasmssa.const -157.5 : f64
+// CHECK:           return %[[VAL_0]], %[[VAL_1]], %[[VAL_2]], %[[VAL_3]] : i32, i64, f32, f64
+wasmssa.return %0, %1, %2, %3 : i32, i64, f32, f64
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-convert-to-arith-tofp.mlir b/mlir/test/Conversion/RaiseWasm/wasm-convert-to-arith-tofp.mlir
new file mode 100644
index 0000000000000..4eeea757d48d0
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-convert-to-arith-tofp.mlir
@@ -0,0 +1,81 @@
+// RUN: mlir-opt %s --raise-wasm-mlir | FileCheck %s
+
+// CHECK-LABEL:   func.func @convert_i32_u_to_f32() -> f32 {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 10 : i32
+// CHECK:           %[[VAL_1:.*]] = arith.uitofp %[[VAL_0]] : i32 to f32
+// CHECK:           return %[[VAL_1]] : f32
+wasmssa.func @convert_i32_u_to_f32() -> f32 {
+  %0 = wasmssa.const 10 : i32
+  %1 = wasmssa.convert_u %0 : i32 to f32
+  wasmssa.return %1 : f32
+}
+
+// CHECK-LABEL:   func.func @convert_i32_s_to_f32() -> f32 {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 42 : i32
+// CHECK:           %[[VAL_1:.*]] = arith.sitofp %[[VAL_0]] : i32 to f32
+// CHECK:           return %[[VAL_1]] : f32
+wasmssa.func @convert_i32_s_to_f32() -> f32 {
+  %0 = wasmssa.const 42 : i32
+  %1 = wasmssa.convert_s %0 : i32 to f32
+  wasmssa.return %1 : f32
+}
+
+// CHECK-LABEL:   func.func @convert_i64_u_to_f32() -> f32 {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 17 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.uitofp %[[VAL_0]] : i64 to f32
+// CHECK:           return %[[VAL_1]] : f32
+wasmssa.func @convert_i64_u_to_f32() -> f32 {
+  %0 = wasmssa.const 17 : i64
+  %1 = wasmssa.convert_u %0 : i64 to f32
+  wasmssa.return %1 : f32
+}
+
+// CHECK-LABEL:   func.func @convert_i64s_to_f32() -> f32 {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 10 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.sitofp %[[VAL_0]] : i64 to f32
+// CHECK:           return %[[VAL_1]] : f32
+wasmssa.func @convert_i64s_to_f32() -> f32 {
+  %0 = wasmssa.const 10 : i64
+  %1 = wasmssa.convert_s %0 : i64 to f32
+  wasmssa.return %1 : f32
+}
+
+// CHECK-LABEL:   func.func @convert_i32_u_to_f64() -> f64 {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 10 : i32
+// CHECK:           %[[VAL_1:.*]] = arith.uitofp %[[VAL_0]] : i32 to f64
+// CHECK:           return %[[VAL_1]] : f64
+wasmssa.func @convert_i32_u_to_f64() -> f64 {
+  %0 = wasmssa.const 10 : i32
+  %1 = wasmssa.convert_u %0 : i32 to f64
+  wasmssa.return %1 : f64
+}
+
+// CHECK-LABEL:   func.func @convert_i32_s_to_f64() -> f64 {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 42 : i32
+// CHECK:           %[[VAL_1:.*]] = arith.sitofp %[[VAL_0]] : i32 to f64
+// CHECK:           return %[[VAL_1]] : f64
+wasmssa.func @convert_i32_s_to_f64() -> f64 {
+  %0 = wasmssa.const 42 : i32
+  %1 = wasmssa.convert_s %0 : i32 to f64
+  wasmssa.return %1 : f64
+}
+
+// CHECK-LABEL:   func.func @convert_i64_u_to_f64() -> f64 {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 17 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.uitofp %[[VAL_0]] : i64 to f64
+// CHECK:           return %[[VAL_1]] : f64
+wasmssa.func @convert_i64_u_to_f64() -> f64 {
+  %0 = wasmssa.const 17 : i64
+  %1 = wasmssa.convert_u %0 : i64 to f64
+  wasmssa.return %1 : f64
+}
+
+// CHECK-LABEL:   func.func @convert_i64s_to_f64() -> f64 {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 10 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.sitofp %[[VAL_0]] : i64 to f64
+// CHECK:           return %[[VAL_1]] : f64
+wasmssa.func @convert_i64s_to_f64() -> f64 {
+  %0 = wasmssa.const 10 : i64
+  %1 = wasmssa.convert_s %0 : i64 to f64
+  wasmssa.return %1 : f64
+}
\ No newline at end of file
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-copysign-to-math-copysign.mlir b/mlir/test/Conversion/RaiseWasm/wasm-copysign-to-math-copysign.mlir
new file mode 100644
index 0000000000000..9ab2b7990e7c0
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-copysign-to-math-copysign.mlir
@@ -0,0 +1,38 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir -o - | FileCheck %s
+
+
+// CHECK-LABEL:   func.func @copysign_f32(
+// CHECK-SAME:      %[[ARG0:.*]]: f32,
+// CHECK-SAME:      %[[ARG1:.*]]: f32) -> f32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<f32>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<f32>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<f32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<f32>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<f32>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<f32>
+// CHECK:           %[[VAL_4:.*]] = math.copysign %[[VAL_2]], %[[VAL_3]] : f32
+// CHECK:           return %[[VAL_4]] : f32
+wasmssa.func exported @copysign_f32(%arg0: !wasmssa<local ref to f32>, %arg1: !wasmssa<local ref to f32>) -> f32 {
+    %v0 = wasmssa.local_get %arg0 : ref to f32
+    %v1 = wasmssa.local_get %arg1 : ref to f32
+    %op = wasmssa.copysign %v0 %v1: f32
+    wasmssa.return %op : f32
+}
+
+// CHECK-LABEL:   func.func @copysign_f64(
+// CHECK-SAME:      %[[ARG0:.*]]: f64,
+// CHECK-SAME:      %[[ARG1:.*]]: f64) -> f64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<f64>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<f64>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<f64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<f64>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<f64>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<f64>
+// CHECK:           %[[VAL_4:.*]] = math.copysign %[[VAL_2]], %[[VAL_3]] : f64
+// CHECK:           return %[[VAL_4]] : f64
+wasmssa.func exported @copysign_f64(%arg0: !wasmssa<local ref to f64>, %arg1: !wasmssa<local ref to f64>) -> f64 {
+    %v0 = wasmssa.local_get %arg0 : ref to f64
+    %v1 = wasmssa.local_get %arg1 : ref to f64
+    %op = wasmssa.copysign %v0 %v1: f64
+    wasmssa.return %op : f64
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-ctz-to-math-ctz.mlir b/mlir/test/Conversion/RaiseWasm/wasm-ctz-to-math-ctz.mlir
new file mode 100644
index 0000000000000..6cc9394eea99c
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-ctz-to-math-ctz.mlir
@@ -0,0 +1,27 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir -o - | FileCheck %s
+
+// CHECK-LABEL:   func.func @ctz_i32(
+// CHECK-SAME:      %[[ARG0:.*]]: i32) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_1:.*]] = memref.load %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_2:.*]] = math.cttz %[[VAL_1]] : i32
+// CHECK:           return %[[VAL_2]] : i32
+wasmssa.func exported @ctz_i32(%arg0: !wasmssa<local ref to i32>) -> i32 {
+    %v0 = wasmssa.local_get %arg0 : ref to i32
+    %op = wasmssa.ctz %v0 : i32
+    wasmssa.return %op : i32
+}
+
+// CHECK-LABEL:   func.func @ctz_i64(
+// CHECK-SAME:      %[[ARG0:.*]]: i64) -> i64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_1:.*]] = memref.load %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_2:.*]] = math.cttz %[[VAL_1]] : i64
+// CHECK:           return %[[VAL_2]] : i64
+wasmssa.func exported @ctz_i64(%arg0: !wasmssa<local ref to i64>) -> i64 {
+    %v0 = wasmssa.local_get %arg0 : ref to i64
+    %op = wasmssa.ctz %v0 : i64
+    wasmssa.return %op : i64
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-demote-to-arith-trunc.mlir b/mlir/test/Conversion/RaiseWasm/wasm-demote-to-arith-trunc.mlir
new file mode 100644
index 0000000000000..342886e3bda00
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-demote-to-arith-trunc.mlir
@@ -0,0 +1,14 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir -o - | FileCheck %s
+
+module {
+  wasmssa.func @func_0() -> f32 {
+    %0 = wasmssa.const 2.240000e+00 : f64
+    %1 = wasmssa.demote %0 : f64 to f32
+    wasmssa.return %1 : f32
+  }
+}
+
+// CHECK-LABEL:   func.func @func_0() -> f32 {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 2.240000e+00 : f64
+// CHECK:           %[[VAL_1:.*]] = arith.truncf %[[VAL_0]] : f64 to f32
+// CHECK:           return %[[VAL_1]] : f32
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-div-to-arith-div.mlir b/mlir/test/Conversion/RaiseWasm/wasm-div-to-arith-div.mlir
new file mode 100644
index 0000000000000..3d6fa9af95442
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-div-to-arith-div.mlir
@@ -0,0 +1,109 @@
+// RUN: mlir-opt %s --raise-wasm-mlir | FileCheck %s
+
+// CHECK-LABEL:   func.func @div_i32_si(
+// CHECK-SAME:      %[[ARG0:.*]]: i32,
+// CHECK-SAME:      %[[ARG1:.*]]: i32) -> i32 {
+wasmssa.func @div_i32_si(%arg0: !wasmssa<local ref to i32>, %arg1: !wasmssa<local ref to i32>) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i32>
+%v0 = wasmssa.local_get %arg0 : ref to i32
+%v1 = wasmssa.local_get %arg1 : ref to i32
+// CHECK:           %[[VAL_4:.*]] = arith.divsi %[[VAL_2]], %[[VAL_3]] : i32
+%0 = wasmssa.div_si %v0 %v1 : i32
+// CHECK:           return %[[VAL_4]] : i32
+wasmssa.return %0 : i32
+}
+
+// CHECK-LABEL:   func.func @div_i64_si(
+// CHECK-SAME:      %[[ARG0:.*]]: i64,
+// CHECK-SAME:      %[[ARG1:.*]]: i64) -> i64 {
+wasmssa.func @div_i64_si(%arg0: !wasmssa<local ref to i64>, %arg1: !wasmssa<local ref to i64>) -> i64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i64>
+%v0 = wasmssa.local_get %arg0 : ref to i64
+%v1 = wasmssa.local_get %arg1 : ref to i64
+// CHECK:           %[[VAL_4:.*]] = arith.divsi %[[VAL_2]], %[[VAL_3]] : i64
+%0 = wasmssa.div_si %v0 %v1 : i64
+// CHECK:           return %[[VAL_4]] : i64
+wasmssa.return %0 : i64
+}
+
+// CHECK-LABEL:   func.func @div_i32_ui(
+// CHECK-SAME:      %[[ARG0:.*]]: i32,
+// CHECK-SAME:      %[[ARG1:.*]]: i32) -> i32 {
+wasmssa.func @div_i32_ui(%arg0: !wasmssa<local ref to i32>, %arg1: !wasmssa<local ref to i32>) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i32>
+%v0 = wasmssa.local_get %arg0 : ref to i32
+%v1 = wasmssa.local_get %arg1 : ref to i32
+// CHECK:           %[[VAL_4:.*]] = arith.divui %[[VAL_2]], %[[VAL_3]] : i32
+%0 = wasmssa.div_ui %v0 %v1 : i32
+// CHECK:           return %[[VAL_4]] : i32
+wasmssa.return %0 : i32
+}
+
+// CHECK-LABEL:   func.func @div_i64_ui(
+// CHECK-SAME:      %[[ARG0:.*]]: i64,
+// CHECK-SAME:      %[[ARG1:.*]]: i64) -> i64 {
+wasmssa.func @div_i64_ui(%arg0: !wasmssa<local ref to i64>, %arg1: !wasmssa<local ref to i64>) -> i64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i64>
+%v0 = wasmssa.local_get %arg0 : ref to i64
+%v1 = wasmssa.local_get %arg1 : ref to i64
+// CHECK:           %[[VAL_4:.*]] = arith.divui %[[VAL_2]], %[[VAL_3]] : i64
+%0 = wasmssa.div_ui %v0 %v1 : i64
+// CHECK:           return %[[VAL_4]] : i64
+wasmssa.return %0 : i64
+}
+
+// CHECK-LABEL:   func.func @div_f32(
+// CHECK-SAME:      %[[ARG0:.*]]: f32,
+// CHECK-SAME:      %[[ARG1:.*]]: f32) -> f32 {
+wasmssa.func @div_f32(%arg0: !wasmssa<local ref to f32>, %arg1: !wasmssa<local ref to f32>) -> f32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<f32>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<f32>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<f32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<f32>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<f32>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<f32>
+%v0 = wasmssa.local_get %arg0 : ref to f32
+%v1 = wasmssa.local_get %arg1 : ref to f32
+// CHECK:           %[[VAL_4:.*]] = arith.divf %[[VAL_2]], %[[VAL_3]] : f32
+%0 = wasmssa.div %v0 %v1 : f32
+// CHECK:           return %[[VAL_4]] : f32
+wasmssa.return %0 : f32
+}
+
+// CHECK-LABEL:   func.func @div_f64(
+// CHECK-SAME:      %[[ARG0:.*]]: f64,
+// CHECK-SAME:      %[[ARG1:.*]]: f64) -> f64 {
+wasmssa.func @div_f64(%arg0: !wasmssa<local ref to f64>, %arg1: !wasmssa<local ref to f64>) -> f64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<f64>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<f64>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<f64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<f64>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<f64>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<f64>
+%v0 = wasmssa.local_get %arg0 : ref to f64
+%v1 = wasmssa.local_get %arg1 : ref to f64
+// CHECK:           %[[VAL_4:.*]] = arith.divf %[[VAL_2]], %[[VAL_3]] : f64
+%0 = wasmssa.div %v0 %v1 : f64
+// CHECK:           return %[[VAL_4]] : f64
+wasmssa.return %0 : f64
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-floor-to-math-floor.mlir b/mlir/test/Conversion/RaiseWasm/wasm-floor-to-math-floor.mlir
new file mode 100644
index 0000000000000..5ca4694bdb824
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-floor-to-math-floor.mlir
@@ -0,0 +1,24 @@
+// RUN: mlir-opt %s --raise-wasm-mlir | FileCheck %s
+
+module {
+  wasmssa.func @func_0() -> f64 {
+    %0 = wasmssa.const -1.210000e+01 : f64
+    %1 = wasmssa.floor %0 : f64
+    wasmssa.return %1 : f64
+  }
+  wasmssa.func @func_1() -> f32 {
+    %0 = wasmssa.const 1.618000e+00 : f32
+    %1 = wasmssa.floor %0 : f32
+    wasmssa.return %1 : f32
+  }
+}
+
+// CHECK-LABEL:   func.func @func_0() -> f64 {
+// CHECK:           %[[VAL_0:.*]] = arith.constant -1.210000e+01 : f64
+// CHECK:           %[[VAL_1:.*]] = math.floor %[[VAL_0]] : f64
+// CHECK:           return %[[VAL_1]] : f64
+
+// CHECK-LABEL:   func.func @func_1() -> f32 {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1.618000e+00 : f32
+// CHECK:           %[[VAL_1:.*]] = math.floor %[[VAL_0]] : f32
+// CHECK:           return %[[VAL_1]] : f32
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-func-to-func.mlir b/mlir/test/Conversion/RaiseWasm/wasm-func-to-func.mlir
new file mode 100644
index 0000000000000..65a6831b442ca
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-func-to-func.mlir
@@ -0,0 +1,35 @@
+// RUN: mlir-opt %s --raise-wasm-mlir | FileCheck %s
+
+
+// CHECK-LABEL:   func.func @callee(
+// CHECK-SAME:                      %[[ARG0:.*]]: i32) -> i32 {
+wasmssa.func exported @callee(%arg0: !wasmssa<local ref to i32>) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_1:.*]] = memref.load %[[VAL_0]][] : memref<i32>
+%v0 = wasmssa.local_get %arg0 : ref to i32
+// CHECK:           return %[[VAL_1]] : i32
+wasmssa.return %v0 : i32
+}
+
+wasmssa.func exported @caller(%arg0: !wasmssa<local ref to i32>) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_1:.*]] = memref.load %[[VAL_0]][] : memref<i32>
+%v0 = wasmssa.local_get %arg0 : ref to i32
+// CHECK:           %[[VAL_2:.*]] = call @callee(%[[VAL_1]]) : (i32) -> i32
+%0 = wasmssa.call @callee (%v0) : (i32) -> i32
+// CHECK:           return %[[VAL_2]] : i32
+wasmssa.return %0 : i32
+}
+
+// CHECK-LABEL:         func.func private @"my_module::foo"() -> i32
+wasmssa.import_func "foo" from "my_module" as @func_0 {sym_visibility = "nested", type = () -> (i32)}
+
+// CHECK-LABEL:   func.func @user_of_func0() -> i32 {
+wasmssa.func exported @user_of_func0() -> i32 {
+// CHECK:           %[[VAL_0:.*]] = call @"my_module::foo"() : () -> i32
+%0 = wasmssa.call @func_0 : () -> i32
+// CHECK:           return %[[VAL_0]] : i32
+wasmssa.return %0 : i32
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-global-to-memref-global.mlir b/mlir/test/Conversion/RaiseWasm/wasm-global-to-memref-global.mlir
new file mode 100644
index 0000000000000..45a6636d89c22
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-global-to-memref-global.mlir
@@ -0,0 +1,43 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir | FileCheck %s
+
+wasmssa.global @global_i32 i32 mutable : {
+  %0 = wasmssa.const 66560 : i32
+  wasmssa.return %0 : i32
+}
+
+wasmssa.global @global_i64 i64 mutable : {
+  %0 = wasmssa.const 37017 : i64
+  wasmssa.return %0 : i64
+}
+
+wasmssa.global @global_f32 f32 mutable : {
+  %0 = wasmssa.const 0.125 : f32
+  wasmssa.return %0 : f32
+}
+
+wasmssa.global @global_f64 f64 mutable : {
+  %0 = wasmssa.const 3.14 : f64
+  wasmssa.return %0 : f64
+}
+
+wasmssa.global @global_user0 i32 mutable : {
+  %0 = wasmssa.global_get @global_top_define : i32
+  wasmssa.return %0 : i32
+}
+
+wasmssa.import_global "extern_global_var" from "module" as @global_top_define nested : i32
+
+// CHECK-LABEL:   memref.global "private" @global_i32 : memref<1xi32> = dense<66560>
+// CHECK:         memref.global "private" @global_i64 : memref<1xi64> = dense<37017>
+// CHECK:         memref.global "private" @global_f32 : memref<1xf32> = dense<1.250000e-01>
+// CHECK:         memref.global "private" @global_f64 : memref<1xf64> = dense<3.140000e+00>
+// CHECK:         memref.global "private" @global_user0 : memref<1xi32> = uninitialized
+
+// CHECK-LABEL:   func.func @"global_user0::initializer"() attributes {initializer} {
+// CHECK:           %[[VAL_0:.*]] = memref.get_global @"module::extern_global_var" : memref<1xi32>
+// CHECK:           %[[VAL_1:.*]] = memref.get_global @global_user0 : memref<1xi32>
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]]{{\[}}%[[VAL_2]]] : memref<1xi32>
+// CHECK:           memref.store %[[VAL_3]], %[[VAL_1]]{{\[}}%[[VAL_2]]] : memref<1xi32>
+// CHECK:           return
+// CHECK:         memref.global "nested" constant @"module::extern_global_var" : memref<1xi32>
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-local-to-memref.mlir b/mlir/test/Conversion/RaiseWasm/wasm-local-to-memref.mlir
new file mode 100644
index 0000000000000..839975ab87daa
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-local-to-memref.mlir
@@ -0,0 +1,68 @@
+// RUN: mlir-opt %s --raise-wasm-mlir | FileCheck %s
+
+// CHECK-LABEL:   func.func @func_0() -> f32 {
+wasmssa.func exported @func_0() -> f32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<f32>
+// CHECK:           %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           memref.store %[[VAL_1]], %[[VAL_0]][] : memref<f32>
+  %0 = wasmssa.local of type f32
+// CHECK:           %[[VAL_2:.*]] = memref.alloca() : memref<f32>
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           memref.store %[[VAL_3]], %[[VAL_2]][] : memref<f32>
+  %1 = wasmssa.local of type f32
+// CHECK:           %[[VAL_4:.*]] = arith.constant 8.000000e+00 : f32
+  %2 = wasmssa.const 8.000000e+00 : f32
+// CHECK:           memref.store %[[VAL_4]], %[[VAL_0]][] : memref<f32>
+  wasmssa.local_set %0 : ref to f32 to %2 : f32
+// CHECK:           %[[VAL_5:.*]] = memref.load %[[VAL_0]][] : memref<f32>
+  %3 = wasmssa.local_get %0 : ref to f32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 1.200000e+01 : f32
+  %4 = wasmssa.const 1.200000e+01 : f32
+// CHECK:           memref.store %[[VAL_6]], %[[VAL_2]][] : memref<f32>
+  %5 = wasmssa.local_tee %1 : ref to f32 to %4 : f32
+// CHECK:           %[[VAL_7:.*]] = arith.addf %[[VAL_5]], %[[VAL_6]] : f32
+  %6 = wasmssa.add %3 %5 : f32
+// CHECK:           return %[[VAL_7]] : f32
+  wasmssa.return %6 : f32
+}
+
+// CHECK-LABEL:   func.func @func_1() -> i32 {
+wasmssa.func exported @func_1() -> i32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i32>
+// CHECK:           %[[VAL_1:.*]] = arith.constant 0 : i32
+// CHECK:           memref.store %[[VAL_1]], %[[VAL_0]][] : memref<i32>
+  %0 = wasmssa.local of type i32
+// CHECK:           %[[VAL_2:.*]] = memref.alloca() : memref<i32>
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : i32
+// CHECK:           memref.store %[[VAL_3]], %[[VAL_2]][] : memref<i32>
+  %1 = wasmssa.local of type i32
+// CHECK:           %[[VAL_4:.*]] = arith.constant 8 : i32
+  %2 = wasmssa.const 8 : i32
+// CHECK:           memref.store %[[VAL_4]], %[[VAL_0]][] : memref<i32>
+  wasmssa.local_set %0 : ref to i32 to %2 : i32
+// CHECK:           %[[VAL_5:.*]] = memref.load %[[VAL_0]][] : memref<i32>
+  %3 = wasmssa.local_get %0 : ref to i32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 12 : i32
+  %4 = wasmssa.const 12 : i32
+// CHECK:           memref.store %[[VAL_6]], %[[VAL_2]][] : memref<i32>
+  %5 = wasmssa.local_tee %1 : ref to i32 to %4 : i32
+// CHECK:           %[[VAL_7:.*]] = arith.addi %[[VAL_5]], %[[VAL_6]] : i32
+  %6 = wasmssa.add %3 %5 : i32
+// CHECK:           return %[[VAL_7]] : i32
+  wasmssa.return %6 : i32
+}
+
+// CHECK-LABEL:   func.func @func_2(
+// CHECK-SAME:                      %[[VAL_0:.*]]: i32) -> i32 {
+wasmssa.func exported @func_2(%arg0: !wasmssa<local ref to i32>) -> i32 {
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[VAL_0]], %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_2:.*]] = arith.constant 3 : i32
+  %1 = wasmssa.const 3 : i32
+// CHECK:           memref.store %[[VAL_2]], %[[VAL_1]][] : memref<i32>
+  wasmssa.local_set %arg0 : ref to i32 to %1 : i32
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_1]][] : memref<i32>
+  %2 = wasmssa.local_get %arg0 : ref to i32
+// CHECK:           return %[[VAL_3]] : i32
+  wasmssa.return %2 : i32
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-max-to-arith-maximumf.mlir b/mlir/test/Conversion/RaiseWasm/wasm-max-to-arith-maximumf.mlir
new file mode 100644
index 0000000000000..2f0d10788f03c
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-max-to-arith-maximumf.mlir
@@ -0,0 +1,37 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir -o - | FileCheck %s
+
+// CHECK-LABEL:   func.func @max_f32(
+// CHECK-SAME:      %[[ARG0:.*]]: f32,
+// CHECK-SAME:      %[[ARG1:.*]]: f32) -> f32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<f32>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<f32>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<f32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<f32>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<f32>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<f32>
+// CHECK:           %[[VAL_4:.*]] = arith.maximumf %[[VAL_2]], %[[VAL_3]] : f32
+// CHECK:           return %[[VAL_4]] : f32
+wasmssa.func exported @max_f32(%arg0: !wasmssa<local ref to f32>, %arg1: !wasmssa<local ref to f32>) -> f32 {
+    %v0 = wasmssa.local_get %arg0 : ref to f32
+    %v1 = wasmssa.local_get %arg1 : ref to f32
+    %op = wasmssa.max %v0 %v1 : f32
+    wasmssa.return %op : f32
+}
+
+// CHECK-LABEL:   func.func @max_f64(
+// CHECK-SAME:      %[[ARG0:.*]]: f64,
+// CHECK-SAME:      %[[ARG1:.*]]: f64) -> f64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<f64>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<f64>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<f64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<f64>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<f64>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<f64>
+// CHECK:           %[[VAL_4:.*]] = arith.maximumf %[[VAL_2]], %[[VAL_3]] : f64
+// CHECK:           return %[[VAL_4]] : f64
+wasmssa.func exported @max_f64(%arg0: !wasmssa<local ref to f64>, %arg1: !wasmssa<local ref to f64>) -> f64 {
+    %v0 = wasmssa.local_get %arg0 : ref to f64
+    %v1 = wasmssa.local_get %arg1 : ref to f64
+    %op = wasmssa.max %v0 %v1 : f64
+    wasmssa.return %op : f64
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-min-to-arith-minimumf.mlir b/mlir/test/Conversion/RaiseWasm/wasm-min-to-arith-minimumf.mlir
new file mode 100644
index 0000000000000..d6a0141a5199f
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-min-to-arith-minimumf.mlir
@@ -0,0 +1,37 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir -o - | FileCheck %s
+
+// CHECK-LABEL:   func.func @min_f32(
+// CHECK-SAME:      %[[ARG0:.*]]: f32,
+// CHECK-SAME:      %[[ARG1:.*]]: f32) -> f32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<f32>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<f32>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<f32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<f32>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<f32>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<f32>
+// CHECK:           %[[VAL_4:.*]] = arith.minimumf %[[VAL_2]], %[[VAL_3]] : f32
+// CHECK:           return %[[VAL_4]] : f32
+wasmssa.func exported @min_f32(%arg0: !wasmssa<local ref to f32>, %arg1: !wasmssa<local ref to f32>) -> f32 {
+    %v0 = wasmssa.local_get %arg0 : ref to f32
+    %v1 = wasmssa.local_get %arg1 : ref to f32
+    %op = wasmssa.min %v0 %v1 : f32
+    wasmssa.return %op : f32
+}
+
+// CHECK-LABEL:   func.func @min_f64(
+// CHECK-SAME:      %[[ARG0:.*]]: f64,
+// CHECK-SAME:      %[[ARG1:.*]]: f64) -> f64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<f64>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<f64>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<f64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<f64>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<f64>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<f64>
+// CHECK:           %[[VAL_4:.*]] = arith.minimumf %[[VAL_2]], %[[VAL_3]] : f64
+// CHECK:           return %[[VAL_4]] : f64
+wasmssa.func exported @min_f64(%arg0: !wasmssa<local ref to f64>, %arg1: !wasmssa<local ref to f64>) -> f64 {
+    %v0 = wasmssa.local_get %arg0 : ref to f64
+    %v1 = wasmssa.local_get %arg1 : ref to f64
+    %op = wasmssa.min %v0 %v1 : f64
+    wasmssa.return %op : f64
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-mul-to-arith-mul.mlir b/mlir/test/Conversion/RaiseWasm/wasm-mul-to-arith-mul.mlir
new file mode 100644
index 0000000000000..51c8b43d980b5
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-mul-to-arith-mul.mlir
@@ -0,0 +1,78 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir | FileCheck %s
+
+// CHECK-LABEL:   func.func @mul_i32(
+// CHECK-SAME:                      %[[ARG0:.*]]: i32,
+// CHECK-SAME:                      %[[ARG1:.*]]: i32) -> i32 {
+wasmssa.func @mul_i32(%arg0: !wasmssa<local ref to i32>, %arg1: !wasmssa<local ref to i32>) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i32>
+%v0 = wasmssa.local_get %arg0 : ref to i32
+%v1 = wasmssa.local_get %arg1 : ref to i32
+// CHECK:           %[[VAL_4:.*]] = arith.muli %[[VAL_2]], %[[VAL_3]] : i32
+%0 = wasmssa.mul %v0 %v1 : i32
+// CHECK:           return %[[VAL_4]] : i32
+wasmssa.return %0 : i32
+}
+
+// -----
+
+// CHECK-LABEL:   func.func @mul_i64(
+// CHECK-SAME:                      %[[ARG0:.*]]: i64,
+// CHECK-SAME:                      %[[ARG1:.*]]: i64) -> i64 {
+wasmssa.func @mul_i64(%arg0: !wasmssa<local ref to i64>, %arg1: !wasmssa<local ref to i64>) -> i64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i64>
+%v0 = wasmssa.local_get %arg0 : ref to i64
+%v1 = wasmssa.local_get %arg1 : ref to i64
+// CHECK:           %[[VAL_4:.*]] = arith.muli %[[VAL_2]], %[[VAL_3]] : i64
+%0 = wasmssa.mul %v0 %v1 : i64
+// CHECK:           return %[[VAL_4]] : i64
+wasmssa.return %0 : i64
+}
+// -----
+
+// CHECK-LABEL:   func.func @mul_f32(
+// CHECK-SAME:                      %[[ARG0:.*]]: f32,
+// CHECK-SAME:                      %[[ARG1:.*]]: f32) -> f32 {
+wasmssa.func @mul_f32(%arg0: !wasmssa<local ref to f32>, %arg1: !wasmssa<local ref to f32>) -> f32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<f32>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<f32>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<f32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<f32>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<f32>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<f32>
+%v0 = wasmssa.local_get %arg0 : ref to f32
+%v1 = wasmssa.local_get %arg1 : ref to f32
+// CHECK:           %[[VAL_4:.*]] = arith.mulf %[[VAL_2]], %[[VAL_3]] : f32
+%0 = wasmssa.mul %v0 %v1 : f32
+// CHECK:           return %[[VAL_4]] : f32
+wasmssa.return %0 : f32
+}
+
+// -----
+
+// CHECK-LABEL:   func.func @mul_f64(
+// CHECK-SAME:                      %[[ARG0:.*]]: f64,
+// CHECK-SAME:                      %[[ARG1:.*]]: f64) -> f64 {
+wasmssa.func @mul_f64(%arg0: !wasmssa<local ref to f64>, %arg1: !wasmssa<local ref to f64>) -> f64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<f64>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<f64>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<f64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<f64>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<f64>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<f64>
+%v0 = wasmssa.local_get %arg0 : ref to f64
+%v1 = wasmssa.local_get %arg1 : ref to f64
+// CHECK:           %[[VAL_4:.*]] = arith.mulf %[[VAL_2]], %[[VAL_3]] : f64
+%0 = wasmssa.mul %v0 %v1 : f64
+// CHECK:           return %[[VAL_4]] : f64
+wasmssa.return %0 : f64
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-neg-to-arith-neg.mlir b/mlir/test/Conversion/RaiseWasm/wasm-neg-to-arith-neg.mlir
new file mode 100644
index 0000000000000..a15b9a6c25990
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-neg-to-arith-neg.mlir
@@ -0,0 +1,27 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir -o - | FileCheck %s
+
+// CHECK-LABEL:   func.func @neg_f32(
+// CHECK-SAME:      %[[ARG0:.*]]: f32) -> f32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<f32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_0]][] : memref<f32>
+// CHECK:           %[[VAL_1:.*]] = memref.load %[[VAL_0]][] : memref<f32>
+// CHECK:           %[[VAL_2:.*]] = arith.negf %[[VAL_1]] : f32
+// CHECK:           return %[[VAL_2]] : f32
+wasmssa.func @neg_f32(%arg0: !wasmssa<local ref to f32>) -> f32 {
+    %val = wasmssa.local_get %arg0 : ref to f32
+    %op = wasmssa.neg %val : f32
+    wasmssa.return %op : f32
+}
+
+// CHECK-LABEL:   func.func @neg_f64(
+// CHECK-SAME:      %[[ARG0:.*]]: f64) -> f64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<f64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_0]][] : memref<f64>
+// CHECK:           %[[VAL_1:.*]] = memref.load %[[VAL_0]][] : memref<f64>
+// CHECK:           %[[VAL_2:.*]] = arith.negf %[[VAL_1]] : f64
+// CHECK:           return %[[VAL_2]] : f64
+wasmssa.func @neg_f64(%arg0: !wasmssa<local ref to f64>) -> f64 {
+    %val = wasmssa.local_get %arg0 : ref to f64
+    %op = wasmssa.neg %val : f64
+    wasmssa.return %op : f64
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-or-to-arith-or.mlir b/mlir/test/Conversion/RaiseWasm/wasm-or-to-arith-or.mlir
new file mode 100644
index 0000000000000..44ce35209511d
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-or-to-arith-or.mlir
@@ -0,0 +1,38 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir -o - | FileCheck %s
+
+
+// CHECK-LABEL:   func.func @or_i32(
+// CHECK-SAME:                      %[[ARG0:.*]]: i32,
+// CHECK-SAME:                      %[[ARG1:.*]]: i32) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_4:.*]] = arith.ori %[[VAL_2]], %[[VAL_3]] : i32
+// CHECK:           return %[[VAL_4]] : i32
+wasmssa.func exported @or_i32(%arg0: !wasmssa<local ref to i32>, %arg1: !wasmssa<local ref to i32>) -> i32 {
+    %v0 = wasmssa.local_get %arg0 : ref to i32
+    %v1 = wasmssa.local_get %arg1 : ref to i32
+    %or = wasmssa.or %v0 %v1 : i32
+    wasmssa.return %or : i32
+}
+
+// CHECK-LABEL:   func.func @or_i64(
+// CHECK-SAME:                      %[[ARG0:.*]]: i64,
+// CHECK-SAME:                      %[[ARG1:.*]]: i64) -> i64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_4:.*]] = arith.ori %[[VAL_2]], %[[VAL_3]] : i64
+// CHECK:           return %[[VAL_4]] : i64
+wasmssa.func exported @or_i64(%arg0: !wasmssa<local ref to i64>, %arg1: !wasmssa<local ref to i64>) -> i64 {
+    %v0 = wasmssa.local_get %arg0 : ref to i64
+    %v1 = wasmssa.local_get %arg1 : ref to i64
+    %or = wasmssa.or %v0 %v1 : i64
+    wasmssa.return %or : i64
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-popcnt-to-math-ctpop.mlir b/mlir/test/Conversion/RaiseWasm/wasm-popcnt-to-math-ctpop.mlir
new file mode 100644
index 0000000000000..1c027cf1972e8
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-popcnt-to-math-ctpop.mlir
@@ -0,0 +1,27 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir -o - | FileCheck %s
+
+// CHECK-LABEL:   func.func @popcnt_i32(
+// CHECK-SAME:      %[[ARG0:.*]]: i32) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_1:.*]] = memref.load %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_2:.*]] = math.ctpop %[[VAL_1]] : i32
+// CHECK:           return %[[VAL_2]] : i32
+wasmssa.func exported @popcnt_i32(%arg0: !wasmssa<local ref to i32>) -> i32 {
+    %v = wasmssa.local_get %arg0 : ref to i32
+    %op = wasmssa.popcnt %v : i32
+    wasmssa.return %op : i32
+}
+
+// CHECK-LABEL:   func.func @popcnt_i64(
+// CHECK-SAME:      %[[ARG0:.*]]: i64) -> i64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_1:.*]] = memref.load %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_2:.*]] = math.ctpop %[[VAL_1]] : i64
+// CHECK:           return %[[VAL_2]] : i64
+wasmssa.func exported @popcnt_i64(%arg0: !wasmssa<local ref to i64>) -> i64 {
+    %v = wasmssa.local_get %arg0 : ref to i64
+    %op = wasmssa.popcnt %v : i64
+    wasmssa.return %op : i64
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-promote-to-arith-ext.mlir b/mlir/test/Conversion/RaiseWasm/wasm-promote-to-arith-ext.mlir
new file mode 100644
index 0000000000000..5cf04a0ccbd3c
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-promote-to-arith-ext.mlir
@@ -0,0 +1,11 @@
+// RUN: mlir-opt %s --raise-wasm-mlir | FileCheck %s
+
+// CHECK-LABEL:   func.func @promote_f32_to_f64() -> f64 {
+wasmssa.func @promote_f32_to_f64() -> f64 {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 3.140000e+00 : f32
+  %0 = wasmssa.const 3.14 : f32
+// CHECK:           %[[VAL_1:.*]] = arith.extf %[[VAL_0]] : f32 to f64
+  %1 = wasmssa.promote %0 : f32 to f64
+// CHECK:           return %[[VAL_1]] : f64
+  wasmssa.return %1 : f64
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-reinterpret-to-arith-bitcast.mlir b/mlir/test/Conversion/RaiseWasm/wasm-reinterpret-to-arith-bitcast.mlir
new file mode 100644
index 0000000000000..78e3984870d6b
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-reinterpret-to-arith-bitcast.mlir
@@ -0,0 +1,42 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir -o - | FileCheck %s
+module {
+// CHECK-LABEL:   func.func @i32.reinterpret_f32() -> i32 {
+// CHECK:           %[[VAL_0:.*]] = arith.constant -1.000000e+00 : f32
+// CHECK:           %[[VAL_1:.*]] = arith.bitcast %[[VAL_0]] : f32 to i32
+// CHECK:           return %[[VAL_1]] : i32
+  wasmssa.func @i32.reinterpret_f32() -> i32 {
+    %0 = wasmssa.const -1.000000e+00 : f32
+    %1 = wasmssa.reinterpret %0 : f32 as i32
+    wasmssa.return %1 : i32
+  }
+
+// CHECK-LABEL:   func.func @i64.reinterpret_f64() -> i64 {
+// CHECK:           %[[VAL_0:.*]] = arith.constant -1.000000e+00 : f64
+// CHECK:           %[[VAL_1:.*]] = arith.bitcast %[[VAL_0]] : f64 to i64
+// CHECK:           return %[[VAL_1]] : i64
+  wasmssa.func @i64.reinterpret_f64() -> i64 {
+    %0 = wasmssa.const -1.000000e+00 : f64
+    %1 = wasmssa.reinterpret %0 : f64 as i64
+    wasmssa.return %1 : i64
+  }
+
+// CHECK-LABEL:   func.func @f32.reinterpret_i32() -> f32 {
+// CHECK:           %[[VAL_0:.*]] = arith.constant -1 : i32
+// CHECK:           %[[VAL_1:.*]] = arith.bitcast %[[VAL_0]] : i32 to f32
+// CHECK:           return %[[VAL_1]] : f32
+  wasmssa.func @f32.reinterpret_i32() -> f32 {
+    %0 = wasmssa.const -1 : i32
+    %1 = wasmssa.reinterpret %0 : i32 as f32
+    wasmssa.return %1 : f32
+  }
+
+// CHECK-LABEL:   func.func @f64.reinterpret_i64() -> f64 {
+// CHECK:           %[[VAL_0:.*]] = arith.constant -1 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.bitcast %[[VAL_0]] : i64 to f64
+// CHECK:           return %[[VAL_1]] : f64
+  wasmssa.func @f64.reinterpret_i64() -> f64 {
+    %0 = wasmssa.const -1 : i64
+    %1 = wasmssa.reinterpret %0 : i64 as f64
+    wasmssa.return %1 : f64
+  }
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-rem-to-arith-rem.mlir b/mlir/test/Conversion/RaiseWasm/wasm-rem-to-arith-rem.mlir
new file mode 100644
index 0000000000000..05a5b5bd10cfa
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-rem-to-arith-rem.mlir
@@ -0,0 +1,74 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir -o - | FileCheck %s
+
+
+// CHECK-LABEL:   func.func @rem_ui_32(
+// CHECK-SAME:      %[[ARG0:.*]]: i32,
+// CHECK-SAME:      %[[ARG1:.*]]: i32) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_4:.*]] = arith.remui %[[VAL_2]], %[[VAL_3]] : i32
+// CHECK:           return %[[VAL_4]] : i32
+wasmssa.func exported @rem_ui_32(%arg0: !wasmssa<local ref to i32>, %arg1: !wasmssa<local ref to i32>) -> i32 {
+    %v0 = wasmssa.local_get %arg0: ref to i32
+    %v1 = wasmssa.local_get %arg1: ref to i32
+    %rem = wasmssa.rem_ui %v0 %v1 : i32
+    wasmssa.return %rem : i32
+}
+
+// CHECK-LABEL:   func.func @rem_si_32(
+// CHECK-SAME:      %[[ARG0:.*]]: i32,
+// CHECK-SAME:      %[[ARG1:.*]]: i32) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_4:.*]] = arith.remsi %[[VAL_2]], %[[VAL_3]] : i32
+// CHECK:           return %[[VAL_4]] : i32
+wasmssa.func exported @rem_si_32(%arg0: !wasmssa<local ref to i32>, %arg1: !wasmssa<local ref to i32>) -> i32 {
+    %v0 = wasmssa.local_get %arg0: ref to i32
+    %v1 = wasmssa.local_get %arg1: ref to i32
+    %rem = wasmssa.rem_si %v0 %v1 : i32
+    wasmssa.return %rem : i32
+}
+
+// CHECK-LABEL:   func.func @rem_ui_64(
+// CHECK-SAME:      %[[ARG0:.*]]: i64,
+// CHECK-SAME:      %[[ARG1:.*]]: i64) -> i64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_4:.*]] = arith.remui %[[VAL_2]], %[[VAL_3]] : i64
+// CHECK:           return %[[VAL_4]] : i64
+wasmssa.func exported @rem_ui_64(%arg0: !wasmssa<local ref to i64>, %arg1: !wasmssa<local ref to i64>) -> i64 {
+    %v0 = wasmssa.local_get %arg0: ref to i64
+    %v1 = wasmssa.local_get %arg1: ref to i64
+    %rem = wasmssa.rem_ui %v0 %v1 : i64
+    wasmssa.return %rem : i64
+}
+
+// CHECK-LABEL:   func.func @rem_si_64(
+// CHECK-SAME:      %[[ARG0:.*]]: i64,
+// CHECK-SAME:      %[[ARG1:.*]]: i64) -> i64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_4:.*]] = arith.remsi %[[VAL_2]], %[[VAL_3]] : i64
+// CHECK:           return %[[VAL_4]] : i64
+wasmssa.func exported @rem_si_64(%arg0: !wasmssa<local ref to i64>, %arg1: !wasmssa<local ref to i64>) -> i64 {
+    %v0 = wasmssa.local_get %arg0: ref to i64
+    %v1 = wasmssa.local_get %arg1: ref to i64
+    %rem = wasmssa.rem_si %v0 %v1 : i64
+    wasmssa.return %rem : i64
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-shl-to-arith-shl.mlir b/mlir/test/Conversion/RaiseWasm/wasm-shl-to-arith-shl.mlir
new file mode 100644
index 0000000000000..efc4b5ca49225
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-shl-to-arith-shl.mlir
@@ -0,0 +1,38 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir -o - | FileCheck %s
+
+
+// CHECK-LABEL:   func.func @shl_i32(
+// CHECK-SAME:      %[[ARG0:.*]]: i32,
+// CHECK-SAME:      %[[ARG1:.*]]: i32) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_4:.*]] = arith.shli %[[VAL_2]], %[[VAL_3]] : i32
+// CHECK:           return %[[VAL_4]] : i32
+wasmssa.func exported @shl_i32(%arg0: !wasmssa<local ref to i32>, %arg1: !wasmssa<local ref to i32>) -> i32 {
+    %v0 = wasmssa.local_get %arg0 : ref to i32
+    %v1 = wasmssa.local_get %arg1 : ref to i32
+    %op = wasmssa.shl %v0 by %v1 bits : i32
+    wasmssa.return %op : i32
+}
+
+// CHECK-LABEL:   func.func @shl_i64(
+// CHECK-SAME:      %[[ARG0:.*]]: i64,
+// CHECK-SAME:      %[[ARG1:.*]]: i64) -> i64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_4:.*]] = arith.shli %[[VAL_2]], %[[VAL_3]] : i64
+// CHECK:           return %[[VAL_4]] : i64
+wasmssa.func exported @shl_i64(%arg0: !wasmssa<local ref to i64>, %arg1: !wasmssa<local ref to i64>) -> i64 {
+    %v0 = wasmssa.local_get %arg0 : ref to i64
+    %v1 = wasmssa.local_get %arg1 : ref to i64
+    %op = wasmssa.shl %v0 by %v1 bits : i64
+    wasmssa.return %op : i64
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-shr_s-to-arith-shrs.mlir b/mlir/test/Conversion/RaiseWasm/wasm-shr_s-to-arith-shrs.mlir
new file mode 100644
index 0000000000000..d0c3a94e611a8
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-shr_s-to-arith-shrs.mlir
@@ -0,0 +1,38 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir -o - | FileCheck %s
+
+
+// CHECK-LABEL:   func.func @shr_s_i32(
+// CHECK-SAME:      %[[ARG0:.*]]: i32,
+// CHECK-SAME:      %[[ARG1:.*]]: i32) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_4:.*]] = arith.shrsi %[[VAL_2]], %[[VAL_3]] : i32
+// CHECK:           return %[[VAL_4]] : i32
+wasmssa.func exported @shr_s_i32(%arg0: !wasmssa<local ref to i32>, %arg1: !wasmssa<local ref to i32>) -> i32 {
+    %v0 = wasmssa.local_get %arg0 : ref to i32
+    %v1 = wasmssa.local_get %arg1 : ref to i32
+    %op = wasmssa.shr_s %v0 by %v1 bits : i32
+    wasmssa.return %op : i32
+}
+
+// CHECK-LABEL:   func.func @shr_s_i64(
+// CHECK-SAME:      %[[ARG0:.*]]: i64,
+// CHECK-SAME:      %[[ARG1:.*]]: i64) -> i64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_4:.*]] = arith.shrsi %[[VAL_2]], %[[VAL_3]] : i64
+// CHECK:           return %[[VAL_4]] : i64
+wasmssa.func exported @shr_s_i64(%arg0: !wasmssa<local ref to i64>, %arg1: !wasmssa<local ref to i64>) -> i64 {
+    %v0 = wasmssa.local_get %arg0 : ref to i64
+    %v1 = wasmssa.local_get %arg1 : ref to i64
+    %op = wasmssa.shr_s %v0 by %v1 bits : i64
+    wasmssa.return %op : i64
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-shr_u-to-arith-shru.mlir b/mlir/test/Conversion/RaiseWasm/wasm-shr_u-to-arith-shru.mlir
new file mode 100644
index 0000000000000..38925b6899c4b
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-shr_u-to-arith-shru.mlir
@@ -0,0 +1,37 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir -o - | FileCheck %s
+
+// CHECK-LABEL:   func.func @shr_u_i32(
+// CHECK-SAME:      %[[ARG0:.*]]: i32,
+// CHECK-SAME:      %[[ARG1:.*]]: i32) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_4:.*]] = arith.shrui %[[VAL_2]], %[[VAL_3]] : i32
+// CHECK:           return %[[VAL_4]] : i32
+wasmssa.func exported @shr_u_i32(%arg0: !wasmssa<local ref to i32>, %arg1: !wasmssa<local ref to i32>) -> i32 {
+    %v0 = wasmssa.local_get %arg0 : ref to i32
+    %v1 = wasmssa.local_get %arg1 : ref to i32
+    %op = wasmssa.shr_u %v0 by %v1 bits : i32
+    wasmssa.return %op : i32
+}
+
+// CHECK-LABEL:   func.func @shr_u_i64(
+// CHECK-SAME:      %[[ARG0:.*]]: i64,
+// CHECK-SAME:      %[[ARG1:.*]]: i64) -> i64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_4:.*]] = arith.shrui %[[VAL_2]], %[[VAL_3]] : i64
+// CHECK:           return %[[VAL_4]] : i64
+wasmssa.func exported @shr_u_i64(%arg0: !wasmssa<local ref to i64>, %arg1: !wasmssa<local ref to i64>) -> i64 {
+    %v0 = wasmssa.local_get %arg0 : ref to i64
+    %v1 = wasmssa.local_get %arg1 : ref to i64
+    %op = wasmssa.shr_u %v0 by %v1 bits : i64
+    wasmssa.return %op : i64
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-sqrt-to-math-sqrt.mlir b/mlir/test/Conversion/RaiseWasm/wasm-sqrt-to-math-sqrt.mlir
new file mode 100644
index 0000000000000..394dc9e3216af
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-sqrt-to-math-sqrt.mlir
@@ -0,0 +1,27 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir -o - | FileCheck %s
+
+// CHECK-LABEL:   func.func @sqrt_f32(
+// CHECK-SAME:      %[[ARG0:.*]]: f32) -> f32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<f32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_0]][] : memref<f32>
+// CHECK:           %[[VAL_1:.*]] = memref.load %[[VAL_0]][] : memref<f32>
+// CHECK:           %[[VAL_2:.*]] = math.sqrt %[[VAL_1]] : f32
+// CHECK:           return %[[VAL_2]] : f32
+wasmssa.func @sqrt_f32(%arg0: !wasmssa<local ref to f32>) -> f32 {
+    %local = wasmssa.local_get %arg0 : ref to f32
+    %op = wasmssa.sqrt %local : f32
+    wasmssa.return %op : f32
+}
+
+// CHECK-LABEL:   func.func @sqrt_f64(
+// CHECK-SAME:      %[[ARG0:.*]]: f64) -> f64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<f64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_0]][] : memref<f64>
+// CHECK:           %[[VAL_1:.*]] = memref.load %[[VAL_0]][] : memref<f64>
+// CHECK:           %[[VAL_2:.*]] = math.sqrt %[[VAL_1]] : f64
+// CHECK:           return %[[VAL_2]] : f64
+wasmssa.func @sqrt_f64(%arg0: !wasmssa<local ref to f64>) -> f64 {
+    %local = wasmssa.local_get %arg0 : ref to f64
+    %op = wasmssa.sqrt %local : f64
+    wasmssa.return %op : f64
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-sub-to-arith-sub.mlir b/mlir/test/Conversion/RaiseWasm/wasm-sub-to-arith-sub.mlir
new file mode 100644
index 0000000000000..933d3d16add7b
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-sub-to-arith-sub.mlir
@@ -0,0 +1,80 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir | FileCheck %s
+
+// CHECK-LABEL:   func.func @func_1(
+// CHECK-SAME:                      %[[ARG0:.*]]: i32,
+// CHECK-SAME:                      %[[ARG1:.*]]: i32) -> i32 {
+wasmssa.func @func_1(%arg0: !wasmssa<local ref to i32>, %arg1: !wasmssa<local ref to i32>) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i32>
+%v0 = wasmssa.local_get %arg0 : ref to i32
+%v1 = wasmssa.local_get %arg1 : ref to i32
+// CHECK:           %[[VAL_4:.*]] = arith.subi %[[VAL_2]], %[[VAL_3]] : i32
+%res = wasmssa.sub %v0 %v1 : i32
+// CHECK:           return %[[VAL_4]] : i32
+wasmssa.return %res : i32
+}
+
+// -----
+
+// CHECK-LABEL:   func.func @func_2(
+// CHECK-SAME:                      %[[ARG0:.*]]: i64,
+// CHECK-SAME:                      %[[ARG1:.*]]: i64) -> i64 {
+wasmssa.func @func_2(%arg0: !wasmssa<local ref to i64>, %arg1: !wasmssa<local ref to i64>) -> i64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i64>
+%v0 = wasmssa.local_get %arg0 : ref to i64
+%v1 = wasmssa.local_get %arg1 : ref to i64
+// CHECK:           %[[VAL_4:.*]] = arith.subi %[[VAL_2]], %[[VAL_3]] : i64
+%res = wasmssa.sub %v0 %v1 : i64
+// CHECK:           return %[[VAL_4]] : i64
+wasmssa.return %res : i64
+}
+
+// -----
+
+// CHECK-LABEL:   func.func @func_3(
+// CHECK-SAME:                      %[[ARG0:.*]]: f32,
+// CHECK-SAME:                      %[[ARG1:.*]]: f32) -> f32 {
+wasmssa.func @func_3(%arg0: !wasmssa<local ref to f32>, %arg1: !wasmssa<local ref to f32>) -> f32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<f32>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<f32>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<f32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<f32>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<f32>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<f32>
+%v0 = wasmssa.local_get %arg0 : ref to f32
+%v1 = wasmssa.local_get %arg1 : ref to f32
+// CHECK:           %[[VAL_4:.*]] = arith.subf %[[VAL_2]], %[[VAL_3]] : f32
+%res = wasmssa.sub %v0 %v1 : f32
+// CHECK:           return %[[VAL_4]] : f32
+wasmssa.return %res : f32
+}
+
+
+// -----
+
+// CHECK-LABEL:   func.func @func_4(
+// CHECK-SAME:                      %[[ARG0:.*]]: f64,
+// CHECK-SAME:                      %[[ARG1:.*]]: f64) -> f64 {
+wasmssa.func @func_4(%arg0: !wasmssa<local ref to f64>, %arg1: !wasmssa<local ref to f64>) -> f64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<f64>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<f64>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<f64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<f64>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<f64>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<f64>
+%v0 = wasmssa.local_get %arg0 : ref to f64
+%v1 = wasmssa.local_get %arg1 : ref to f64
+// CHECK:           %[[VAL_4:.*]] = arith.subf %[[VAL_2]], %[[VAL_3]] : f64
+%res = wasmssa.sub %v0 %v1 : f64
+// CHECK:           return %[[VAL_4]] : f64
+wasmssa.return %res : f64
+}
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-trunc-to-math-trunc.mlir b/mlir/test/Conversion/RaiseWasm/wasm-trunc-to-math-trunc.mlir
new file mode 100644
index 0000000000000..a120967bc3c0e
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-trunc-to-math-trunc.mlir
@@ -0,0 +1,24 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir -o - | FileCheck %s
+
+module {
+  wasmssa.func @func_4() -> f64 {
+    %0 = wasmssa.const -1.210000e+01 : f64
+    %1 = wasmssa.trunc %0 : f64
+    wasmssa.return %1 : f64
+  }
+  wasmssa.func @func_5() -> f32 {
+    %0 = wasmssa.const 1.618000e+00 : f32
+    %1 = wasmssa.trunc %0 : f32
+    wasmssa.return %1 : f32
+  }
+}
+
+// CHECK-LABEL:   func.func @func_4() -> f64 {
+// CHECK:           %[[VAL_0:.*]] = arith.constant -1.210000e+01 : f64
+// CHECK:           %[[VAL_1:.*]] = math.trunc %[[VAL_0]] : f64
+// CHECK:           return %[[VAL_1]] : f64
+
+// CHECK-LABEL:   func.func @func_5() -> f32 {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1.618000e+00 : f32
+// CHECK:           %[[VAL_1:.*]] = math.trunc %[[VAL_0]] : f32
+// CHECK:           return %[[VAL_1]] : f32
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-wrap-to-arith-trunc.mlir b/mlir/test/Conversion/RaiseWasm/wasm-wrap-to-arith-trunc.mlir
new file mode 100644
index 0000000000000..6164525626086
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-wrap-to-arith-trunc.mlir
@@ -0,0 +1,17 @@
+// RUN: mlir-opt %s --raise-wasm-mlir | FileCheck %s
+
+module {
+  wasmssa.func @i64_wrap(%arg0: !wasmssa<local ref to i64>) -> i32 {
+    %0 = wasmssa.local_get %arg0 :  ref to i64
+    %1 = wasmssa.wrap %0 : i64 to i32
+    wasmssa.return %1 : i32
+  }
+}
+
+// CHECK-LABEL:   func.func @i64_wrap(
+// CHECK-SAME:      %[[ARG0:.*]]: i64) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_1:.*]] = memref.load %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_2:.*]] = arith.trunci %[[VAL_1]] : i64 to i32
+// CHECK:           return %[[VAL_2]] : i32
diff --git a/mlir/test/Conversion/RaiseWasm/wasm-xor-to-arith-xor.mlir b/mlir/test/Conversion/RaiseWasm/wasm-xor-to-arith-xor.mlir
new file mode 100644
index 0000000000000..810729d883c61
--- /dev/null
+++ b/mlir/test/Conversion/RaiseWasm/wasm-xor-to-arith-xor.mlir
@@ -0,0 +1,37 @@
+// RUN: mlir-opt --split-input-file %s --raise-wasm-mlir -o - | FileCheck %s
+
+// CHECK-LABEL:   func.func @xor_i32(
+// CHECK-SAME:                      %[[ARG0:.*]]: i32,
+// CHECK-SAME:                      %[[ARG1:.*]]: i32) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i32>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i32>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i32>
+// CHECK:           %[[VAL_4:.*]] = arith.xori %[[VAL_2]], %[[VAL_3]] : i32
+// CHECK:           return %[[VAL_4]] : i32
+wasmssa.func @xor_i32(%arg0: !wasmssa<local ref to i32>, %arg1: !wasmssa<local ref to i32>) -> i32 {
+    %v0 = wasmssa.local_get %arg0 : ref to i32
+    %v1 = wasmssa.local_get %arg1 : ref to i32
+    %xor = wasmssa.xor %v0 %v1 : i32
+    wasmssa.return %xor : i32
+}
+
+// CHECK-LABEL:   func.func @xor_i64(
+// CHECK-SAME:                      %[[ARG0:.*]]: i64,
+// CHECK-SAME:                      %[[ARG1:.*]]: i64) -> i64 {
+// CHECK:           %[[VAL_0:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG1]], %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_1:.*]] = memref.alloca() : memref<i64>
+// CHECK:           memref.store %[[ARG0]], %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_2:.*]] = memref.load %[[VAL_1]][] : memref<i64>
+// CHECK:           %[[VAL_3:.*]] = memref.load %[[VAL_0]][] : memref<i64>
+// CHECK:           %[[VAL_4:.*]] = arith.xori %[[VAL_2]], %[[VAL_3]] : i64
+// CHECK:           return %[[VAL_4]] : i64
+wasmssa.func @xor_i64(%arg0: !wasmssa<local ref to i64>, %arg1: !wasmssa<local ref to i64>) -> i64 {
+    %v0 = wasmssa.local_get %arg0 : ref to i64
+    %v1 = wasmssa.local_get %arg1 : ref to i64
+    %xor = wasmssa.xor %v0 %v1 : i64
+    wasmssa.return %xor : i64
+}
diff --git a/mlir/test/Dialect/OpenACC/acc-emit-remarks-private.mlir b/mlir/test/Dialect/OpenACC/acc-emit-remarks-private.mlir
new file mode 100644
index 0000000000000..b388ad51208a2
--- /dev/null
+++ b/mlir/test/Dialect/OpenACC/acc-emit-remarks-private.mlir
@@ -0,0 +1,119 @@
+// RUN: mlir-opt %s -acc-emit-remarks-private --remarks-filter="(open)?acc.*" 2>&1 | FileCheck %s
+
+// CHECK: remark: [Passed] openacc | Category:acc-emit-remarks-private | Function=firstpriv_implicit | Remark="Generating implicit firstprivate(t)"
+// CHECK: remark: [Passed] openacc | Category:acc-emit-remarks-private | Function=firstpriv_explicit | Remark="Generating firstprivate(t)"
+// CHECK: remark: [Passed] openacc | Category:acc-emit-remarks-private | Function=private_loop_implicit | Remark="Generating implicit private(x)"
+// CHECK: remark: [Passed] openacc | Category:acc-emit-remarks-private | Function=private_loop_unknown | Remark="Generating implicit private(<unknown>)"
+// CHECK: remark: [Passed] openacc | Category:acc-emit-remarks-private | Function=private_explicit | Remark="Generating private(x)"
+// CHECK: remark: [Passed] openacc | Category:acc-emit-remarks-private | Function=multi_private_implicit | Remark="Generating implicit private(a,b)"
+
+acc.firstprivate.recipe @firstprivatization_memref_i32 : memref<i32> init {
+^bb0(%arg0: memref<i32>):
+  %0 = memref.alloca() : memref<i32>
+  acc.yield %0 : memref<i32>
+} copy {
+^bb0(%arg0: memref<i32>, %arg1: memref<i32>):
+  %0 = memref.load %arg0[] : memref<i32>
+  memref.store %0, %arg1[] : memref<i32>
+  acc.terminator
+} destroy {
+^bb0(%arg0: memref<i32>, %arg1: memref<i32>):
+  memref.dealloc %arg1 : memref<i32>
+  acc.terminator
+}
+
+acc.private.recipe @privatization_memref_i64 : memref<i64> init {
+^bb0(%arg0: memref<i64>):
+  %0 = memref.alloca() : memref<i64>
+  acc.yield %0 : memref<i64>
+} destroy {
+^bb0(%arg0: memref<i64>, %arg1: memref<i64>):
+  memref.dealloc %arg1 : memref<i64>
+  acc.terminator
+}
+
+// CHECK-LABEL: func.func @firstpriv_implicit
+// CHECK: acc.firstprivate
+// CHECK: acc.parallel firstprivate
+func.func @firstpriv_implicit() {
+  %c1336 = arith.constant 1336 : i32
+  %alloc = memref.alloca() : memref<i32>
+  memref.store %c1336, %alloc[] : memref<i32>
+  %fp = acc.firstprivate varPtr(%alloc : memref<i32>) recipe(@firstprivatization_memref_i32) -> memref<i32> {implicit = true, name = "t"}
+  acc.parallel firstprivate(%fp : memref<i32>) {
+    %c1 = arith.constant 1 : i32
+    %v = memref.load %fp[] : memref<i32>
+    %add = arith.addi %v, %c1 : i32
+    memref.store %add, %fp[] : memref<i32>
+    acc.yield
+  }
+  return
+}
+
+// CHECK-LABEL: func.func @firstpriv_explicit
+func.func @firstpriv_explicit() {
+  %c1336 = arith.constant 1336 : i32
+  %alloc = memref.alloca() : memref<i32>
+  memref.store %c1336, %alloc[] : memref<i32>
+  %fp = acc.firstprivate varPtr(%alloc : memref<i32>) recipe(@firstprivatization_memref_i32) -> memref<i32> {name = "t"}
+  acc.parallel firstprivate(%fp : memref<i32>) {
+    %c1 = arith.constant 1 : i32
+    %v = memref.load %fp[] : memref<i32>
+    %add = arith.addi %v, %c1 : i32
+    memref.store %add, %fp[] : memref<i32>
+    acc.yield
+  }
+  return
+}
+
+// CHECK-LABEL: func.func @private_loop_implicit
+func.func @private_loop_implicit(%arg0 : memref<i64>) {
+  %c16 = arith.constant 16 : index
+  %c1 = arith.constant 1 : index
+  %priv = acc.private varPtr(%arg0 : memref<i64>) recipe(@privatization_memref_i64) -> memref<i64> {implicit = true, name = "x"}
+  acc.loop private(%priv : memref<i64>) control(%siv : index) = (%c1 : index) to (%c16 : index) step (%c1 : index) {
+    %iv_i64 = arith.index_cast %siv : index to i64
+    memref.store %iv_i64, %priv[] : memref<i64>
+    acc.yield
+  } attributes {independent = [#acc.device_type<none>]}
+  return
+}
+
+// CHECK-LABEL: func.func @private_loop_unknown
+func.func @private_loop_unknown(%arg0 : memref<i64>) {
+  %c16 = arith.constant 16 : index
+  %c1 = arith.constant 1 : index
+  %priv = acc.private varPtr(%arg0 : memref<i64>) recipe(@privatization_memref_i64) -> memref<i64> {implicit = true, name = ""}
+  acc.loop private(%priv : memref<i64>) control(%siv : index) = (%c1 : index) to (%c16 : index) step (%c1 : index) {
+    %iv_i64 = arith.index_cast %siv : index to i64
+    memref.store %iv_i64, %priv[] : memref<i64>
+    acc.yield
+  } attributes {independent = [#acc.device_type<none>]}
+  return
+}
+
+// CHECK-LABEL: func.func @private_explicit
+func.func @private_explicit(%arg0 : memref<i64>) {
+  %c16 = arith.constant 16 : index
+  %c1 = arith.constant 1 : index
+  %priv = acc.private varPtr(%arg0 : memref<i64>) recipe(@privatization_memref_i64) -> memref<i64> {name = "x"}
+  acc.parallel private(%priv : memref<i64>) {
+    acc.loop control(%siv : index) = (%c1 : index) to (%c16 : index) step (%c1 : index) {
+      %iv_i64 = arith.index_cast %siv : index to i64
+      memref.store %iv_i64, %priv[] : memref<i64>
+      acc.yield
+    } attributes {independent = [#acc.device_type<none>]}
+    acc.yield
+  }
+  return
+}
+
+// CHECK-LABEL: func.func @multi_private_implicit
+func.func @multi_private_implicit(%arg0 : memref<i64>, %arg1 : memref<i64>) {
+  %privA = acc.private varPtr(%arg0 : memref<i64>) recipe(@privatization_memref_i64) -> memref<i64> {implicit = true, name = "a"}
+  %privB = acc.private varPtr(%arg1 : memref<i64>) recipe(@privatization_memref_i64) -> memref<i64> {implicit = true, name = "b"}
+  acc.parallel private(%privA, %privB : memref<i64>, memref<i64>) {
+    acc.yield
+  }
+  return
+}
diff --git a/mlir/test/Dialect/OpenACC/acc-if-clause-lowering.mlir b/mlir/test/Dialect/OpenACC/acc-if-clause-lowering.mlir
index 4c88df432b6c7..af4cc72a42645 100644
--- a/mlir/test/Dialect/OpenACC/acc-if-clause-lowering.mlir
+++ b/mlir/test/Dialect/OpenACC/acc-if-clause-lowering.mlir
@@ -373,4 +373,36 @@ func.func @test_acc_private(%arg0: memref<i32>, %cond: i1) {
     acc.yield
   }
   return
-}
\ No newline at end of file
+}
+
+// -----
+
+// Test that an acc.parallel with an if clause whose body holds an
+// acc.atomic.capture lowers cleanly: the device path keeps the capture and the
+// host fallback inlines it.
+// CHECK-LABEL: func.func @test_parallel_if_atomic_capture
+func.func @test_parallel_if_atomic_capture(%x: memref<i32>, %v: memref<i32>, %cond: i1) {
+  %c1_i32 = arith.constant 1 : i32
+  // CHECK-NOT: acc.parallel if
+  // CHECK: scf.if %{{.*}} {
+  // CHECK:   acc.parallel {
+  // CHECK:     acc.atomic.capture {
+  // CHECK:   } else {
+  // CHECK-NOT: acc.atomic.capture
+  // CHECK:     memref.load
+  // CHECK:     arith.addi
+  // CHECK:     memref.store
+  // CHECK:   }
+  acc.parallel if(%cond) {
+    acc.atomic.capture {
+      acc.atomic.update %x : memref<i32> {
+      ^bb0(%arg: i32):
+        %r = arith.addi %arg, %c1_i32 : i32
+        acc.yield %r : i32
+      }
+      acc.atomic.read %v = %x : memref<i32>, memref<i32>, i32
+    }
+    acc.yield
+  }
+  return
+}
diff --git a/mlir/test/Dialect/OpenACC/acc-specialize-for-host-deep-nesting.mlir b/mlir/test/Dialect/OpenACC/acc-specialize-for-host-deep-nesting.mlir
new file mode 100644
index 0000000000000..4d7940dbda462
--- /dev/null
+++ b/mlir/test/Dialect/OpenACC/acc-specialize-for-host-deep-nesting.mlir
@@ -0,0 +1,65 @@
+// RUN: mlir-opt %s -acc-specialize-for-host | FileCheck %s
+
+// Deeply nested orphan acc.loops in a seq routine require more than the
+// greedy rewriter's default iteration cap to fully lower to scf.for. The
+// pass must run the rewrite to convergence rather than bailing out.
+
+acc.routine @acc_routine_deep func(@deeply_nested_orphan_loops) seq
+// CHECK-LABEL:   func.func @deeply_nested_orphan_loops
+// CHECK-NOT:       acc.loop
+// CHECK-COUNT-16: scf.for
+func.func @deeply_nested_orphan_loops(%arg0 : memref<i32>) attributes {acc.routine_info = #acc.routine_info<[@acc_routine_deep]>} {
+  %c0 = arith.constant 0 : i32
+  %c10 = arith.constant 10 : i32
+  %c1 = arith.constant 1 : i32
+  acc.loop control(%iv0 : i32) = (%c0 : i32) to (%c10 : i32) step (%c1 : i32) {
+    acc.loop control(%iv1 : i32) = (%c0 : i32) to (%c10 : i32) step (%c1 : i32) {
+      acc.loop control(%iv2 : i32) = (%c0 : i32) to (%c10 : i32) step (%c1 : i32) {
+        acc.loop control(%iv3 : i32) = (%c0 : i32) to (%c10 : i32) step (%c1 : i32) {
+          acc.loop control(%iv4 : i32) = (%c0 : i32) to (%c10 : i32) step (%c1 : i32) {
+            acc.loop control(%iv5 : i32) = (%c0 : i32) to (%c10 : i32) step (%c1 : i32) {
+              acc.loop control(%iv6 : i32) = (%c0 : i32) to (%c10 : i32) step (%c1 : i32) {
+                acc.loop control(%iv7 : i32) = (%c0 : i32) to (%c10 : i32) step (%c1 : i32) {
+                  acc.loop control(%iv8 : i32) = (%c0 : i32) to (%c10 : i32) step (%c1 : i32) {
+                    acc.loop control(%iv9 : i32) = (%c0 : i32) to (%c10 : i32) step (%c1 : i32) {
+                      acc.loop control(%iv10 : i32) = (%c0 : i32) to (%c10 : i32) step (%c1 : i32) {
+                        acc.loop control(%iv11 : i32) = (%c0 : i32) to (%c10 : i32) step (%c1 : i32) {
+                          acc.loop control(%iv12 : i32) = (%c0 : i32) to (%c10 : i32) step (%c1 : i32) {
+                            acc.loop control(%iv13 : i32) = (%c0 : i32) to (%c10 : i32) step (%c1 : i32) {
+                              acc.loop control(%iv14 : i32) = (%c0 : i32) to (%c10 : i32) step (%c1 : i32) {
+                                acc.loop control(%iv15 : i32) = (%c0 : i32) to (%c10 : i32) step (%c1 : i32) {
+                                  memref.store %iv15, %arg0[] : memref<i32>
+                                acc.yield
+                                } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
+                              acc.yield
+                              } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
+                            acc.yield
+                            } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
+                          acc.yield
+                          } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
+                        acc.yield
+                        } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
+                      acc.yield
+                      } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
+                    acc.yield
+                    } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
+                  acc.yield
+                  } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
+                acc.yield
+                } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
+              acc.yield
+              } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
+            acc.yield
+            } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
+          acc.yield
+          } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
+        acc.yield
+        } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
+      acc.yield
+      } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
+    acc.yield
+    } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
+  acc.yield
+  } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
+  return
+}
diff --git a/mlir/test/Dialect/SPIRV/IR/gl-ops.mlir b/mlir/test/Dialect/SPIRV/IR/gl-ops.mlir
index e8f15e30ba4ee..5665ae158d2e4 100644
--- a/mlir/test/Dialect/SPIRV/IR/gl-ops.mlir
+++ b/mlir/test/Dialect/SPIRV/IR/gl-ops.mlir
@@ -1266,3 +1266,57 @@ func.func @degrees(%arg0 : i32) -> () {
   %2 = spirv.GL.Degrees %arg0 : i32
   return
 }
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spirv.GL.NMax
+//===----------------------------------------------------------------------===//
+
+func.func @nmax(%arg0 : f32, %arg1 : f32) {
+  // CHECK: spirv.GL.NMax {{%.*}}, {{%.*}} : f32
+  %0 = spirv.GL.NMax %arg0, %arg1 : f32
+  return
+}
+
+func.func @nmaxvec(%arg0 : vector<3xf16>, %arg1 : vector<3xf16>) {
+  // CHECK: spirv.GL.NMax {{%.*}}, {{%.*}} : vector<3xf16>
+  %0 = spirv.GL.NMax %arg0, %arg1 : vector<3xf16>
+  return
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spirv.GL.NMin
+//===----------------------------------------------------------------------===//
+
+func.func @nmin(%arg0 : f32, %arg1 : f32) {
+  // CHECK: spirv.GL.NMin {{%.*}}, {{%.*}} : f32
+  %0 = spirv.GL.NMin %arg0, %arg1 : f32
+  return
+}
+
+func.func @nminvec(%arg0 : vector<3xf16>, %arg1 : vector<3xf16>) {
+  // CHECK: spirv.GL.NMin {{%.*}}, {{%.*}} : vector<3xf16>
+  %0 = spirv.GL.NMin %arg0, %arg1 : vector<3xf16>
+  return
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spirv.GL.NClamp
+//===----------------------------------------------------------------------===//
+
+func.func @nclamp(%arg0 : f32, %min : f32, %max : f32) -> () {
+  // CHECK: spirv.GL.NClamp {{%[^,]*}}, {{%[^,]*}}, {{%[^,]*}} : f32
+  %2 = spirv.GL.NClamp %arg0, %min, %max : f32
+  return
+}
+
+func.func @nclampvec(%arg0 : vector<3xf32>, %min : vector<3xf32>, %max : vector<3xf32>) -> () {
+  // CHECK: spirv.GL.NClamp {{%[^,]*}}, {{%[^,]*}}, {{%[^,]*}} : vector<3xf32>
+  %2 = spirv.GL.NClamp %arg0, %min, %max : vector<3xf32>
+  return
+}
diff --git a/mlir/test/Dialect/SPIRV/IR/ocl-ops.mlir b/mlir/test/Dialect/SPIRV/IR/ocl-ops.mlir
index e57da6efb13a6..da470f4e08109 100644
--- a/mlir/test/Dialect/SPIRV/IR/ocl-ops.mlir
+++ b/mlir/test/Dialect/SPIRV/IR/ocl-ops.mlir
@@ -860,6 +860,26 @@ func.func @rootn_wrong_type_vec(%arg0 : vector<3xf32>, %arg1 : vector<2xi32>) ->
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// spirv.CL.copysign
+//===----------------------------------------------------------------------===//
+
+func.func @copysign(%arg0 : f32, %arg1 : f32) -> () {
+  // CHECK: spirv.CL.copysign {{%.*}}, {{%.*}} : f32
+  %2 = spirv.CL.copysign %arg0, %arg1 : f32
+  return
+}
+
+// -----
+
+func.func @copysign(%arg0 : vector<4xf16>, %arg1 : vector<4xf16>) -> () {
+  // CHECK: spirv.CL.copysign {{%.*}}, {{%.*}} : vector<4xf16>
+  %2 = spirv.CL.copysign %arg0, %arg1 : vector<4xf16>
+  return
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.CL.expm1
 //===----------------------------------------------------------------------===//
@@ -880,6 +900,66 @@ func.func @expm1(%arg0 : vector<4xf16>) -> () {
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// spirv.CL.fdim
+//===----------------------------------------------------------------------===//
+
+func.func @fdim(%arg0 : f32, %arg1 : f32) -> () {
+  // CHECK: spirv.CL.fdim {{%.*}}, {{%.*}} : f32
+  %2 = spirv.CL.fdim %arg0, %arg1 : f32
+  return
+}
+
+// -----
+
+func.func @fdim(%arg0 : vector<4xf16>, %arg1 : vector<4xf16>) -> () {
+  // CHECK: spirv.CL.fdim {{%.*}}, {{%.*}} : vector<4xf16>
+  %2 = spirv.CL.fdim %arg0, %arg1 : vector<4xf16>
+  return
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spirv.CL.fmod
+//===----------------------------------------------------------------------===//
+
+func.func @fmod(%arg0 : f32, %arg1 : f32) -> () {
+  // CHECK: spirv.CL.fmod {{%.*}}, {{%.*}} : f32
+  %2 = spirv.CL.fmod %arg0, %arg1 : f32
+  return
+}
+
+// -----
+
+func.func @fmod(%arg0 : vector<4xf16>, %arg1 : vector<4xf16>) -> () {
+  // CHECK: spirv.CL.fmod {{%.*}}, {{%.*}} : vector<4xf16>
+  %2 = spirv.CL.fmod %arg0, %arg1 : vector<4xf16>
+  return
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spirv.CL.hypot
+//===----------------------------------------------------------------------===//
+
+func.func @hypot(%arg0 : f32, %arg1 : f32) -> () {
+  // CHECK: spirv.CL.hypot {{%.*}}, {{%.*}} : f32
+  %2 = spirv.CL.hypot %arg0, %arg1 : f32
+  return
+}
+
+// -----
+
+func.func @hypot(%arg0 : vector<4xf16>, %arg1 : vector<4xf16>) -> () {
+  // CHECK: spirv.CL.hypot {{%.*}}, {{%.*}} : vector<4xf16>
+  %2 = spirv.CL.hypot %arg0, %arg1 : vector<4xf16>
+  return
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.CL.log1p
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Target/SPIRV/gl-ops.mlir b/mlir/test/Target/SPIRV/gl-ops.mlir
index eac859ecf158e..bc17ff82cc441 100644
--- a/mlir/test/Target/SPIRV/gl-ops.mlir
+++ b/mlir/test/Target/SPIRV/gl-ops.mlir
@@ -76,6 +76,11 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, Linkage], []> {
     %5 = spirv.GL.SMin %arg2, %arg3 : i32
     // CHECK: {{%.*}} = spirv.GL.UMin {{%.*}}, {{%.*}} : i32
     %6 = spirv.GL.UMin %arg2, %arg3 : i32
+
+    // CHECK: {{%.*}} = spirv.GL.NMax {{%.*}}, {{%.*}} : f32
+    %7 = spirv.GL.NMax %arg0, %arg1 : f32
+    // CHECK: {{%.*}} = spirv.GL.NMin {{%.*}}, {{%.*}} : f32
+    %8 = spirv.GL.NMin %arg0, %arg1 : f32
     spirv.Return
   }
 
@@ -85,6 +90,12 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, Linkage], []> {
     spirv.Return
   }
 
+  spirv.func @nclamp(%arg0 : f32, %arg1 : f32, %arg2 : f32) "None" {
+    // CHECK: spirv.GL.NClamp {{%[^,]*}}, {{%[^,]*}}, {{%[^,]*}} : f32
+    %13 = spirv.GL.NClamp %arg0, %arg1, %arg2 : f32
+    spirv.Return
+  }
+
   spirv.func @uclamp(%arg0 : ui32, %arg1 : ui32, %arg2 : ui32) "None" {
     // CHECK: spirv.GL.UClamp {{%[^,]*}}, {{%[^,]*}}, {{%[^,]*}} : i32
     %13 = spirv.GL.UClamp %arg0, %arg1, %arg2 : ui32
diff --git a/mlir/test/Target/SPIRV/ocl-ops.mlir b/mlir/test/Target/SPIRV/ocl-ops.mlir
index 3ece17e309af2..3359dfd4f0428 100644
--- a/mlir/test/Target/SPIRV/ocl-ops.mlir
+++ b/mlir/test/Target/SPIRV/ocl-ops.mlir
@@ -43,8 +43,16 @@ spirv.module Physical64 OpenCL requires #spirv.vce<v1.0, [Kernel, Addresses, Vec
     %11 = spirv.CL.trunc %arg0 : f32
     // CHECK: {{%.*}} = spirv.CL.cbrt {{%.*}} : f32
     %12 = spirv.CL.cbrt %arg0 : f32
+    // CHECK: {{%.*}} = spirv.CL.copysign {{%.*}}, {{%.*}} : f32
+    %copysign = spirv.CL.copysign %arg0, %arg0 : f32
     // CHECK: {{%.*}} = spirv.CL.expm1 {{%.*}} : f32
     %expm1 = spirv.CL.expm1 %arg0 : f32
+    // CHECK: {{%.*}} = spirv.CL.fdim {{%.*}}, {{%.*}} : f32
+    %fdim = spirv.CL.fdim %arg0, %arg0 : f32
+    // CHECK: {{%.*}} = spirv.CL.fmod {{%.*}}, {{%.*}} : f32
+    %fmod = spirv.CL.fmod %arg0, %arg0 : f32
+    // CHECK: {{%.*}} = spirv.CL.hypot {{%.*}}, {{%.*}} : f32
+    %hypot = spirv.CL.hypot %arg0, %arg0 : f32
     // CHECK: {{%.*}} = spirv.CL.log1p {{%.*}} : f32
     %log1p = spirv.CL.log1p %arg0 : f32
     spirv.Return
diff --git a/offload/test/offloading/fortran/target-no-loop.f90 b/offload/test/offloading/fortran/target-no-loop.f90
index 93b4a1bba7bc6..bd174806ff449 100644
--- a/offload/test/offloading/fortran/target-no-loop.f90
+++ b/offload/test/offloading/fortran/target-no-loop.f90
@@ -88,7 +88,9 @@ end program main
 ! CHECK:  info: #Args: 3 Teams x Thrds:   3x  16 {{.*}}
 ! CHECK:  PluginInterface device {{[0-9]+}} info: Launching kernel {{.*}} SPMD-No-Loop mode
 ! CHECK:  info: #Args: 3 Teams x Thrds:   64x  16 {{.*}}
-! CHECK:  PluginInterface device {{[0-9]+}} info: Launching kernel {{.*}} SPMD mode
+! TODO: This kernel is currently incorrectly tagged as no-loop. This is a known issue in the
+!       Flang frontend that currently does not result in any known runtime problems.
+! CHECK:  PluginInterface device {{[0-9]+}} info: Launching kernel {{.*}} SPMD-No-Loop mode
 ! CHECK:  info: #Args: 3 Teams x Thrds:   1x  16
 ! CHECK:  PluginInterface device {{[0-9]+}} info: Launching kernel {{.*}} Generic mode
 ! CHECK:  info: #Args: 3 Teams x Thrds:   16x  16 {{.*}}
diff --git a/openmp/module/CMakeLists.txt b/openmp/module/CMakeLists.txt
index 451cbf9cc8fc8..f905cfbde0a04 100644
--- a/openmp/module/CMakeLists.txt
+++ b/openmp/module/CMakeLists.txt
@@ -33,13 +33,6 @@ if ("${LLVM_DEFAULT_TARGET_TRIPLE}" MATCHES "^amdgcn|^nvptx")
 endif ()
 
 flang_module_target(libomp-mod PUBLIC)
-add_dependencies(libomp-mod ${RUNTIMES_FORTRAN_BUILD_DEPS})
-if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
-  set_property(SOURCE "${CMAKE_CURRENT_BINARY_DIR}/omp_lib.F90"
-    APPEND PROPERTY OBJECT_DEPENDS
-      "${CMAKE_BINARY_DIR}/modules/${CMAKE_CFG_INTDIR}/__fortran_builtins${CMAKE_Fortran_OUTPUT_EXTENSION}"
-  )
-endif ()
 
 install(FILES
   "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}/omp_lib.h"
diff --git a/openmp/module/omp_lib.F90.var b/openmp/module/omp_lib.F90.var
index 962f3ffd481e4..7055af872771c 100644
--- a/openmp/module/omp_lib.F90.var
+++ b/openmp/module/omp_lib.F90.var
@@ -12,7 +12,7 @@
 
       module omp_lib_kinds
 
-        use, intrinsic :: iso_c_binding
+        use :: iso_c_binding
 
         ! Set PRIVATE by default to explicitly only export what is meant
         ! to be exported by this MODULE.
@@ -672,20 +672,20 @@
           end subroutine omp_display_env
 
           function omp_target_alloc(size, device_num) bind(c)
-            use, intrinsic :: iso_c_binding, only: c_ptr, c_size_t, c_int
+            use :: iso_c_binding, only: c_ptr, c_size_t, c_int
             type(c_ptr) omp_target_alloc
             integer(c_size_t), value :: size
             integer(c_int), value :: device_num
           end function omp_target_alloc
 
           subroutine omp_target_free(device_ptr, device_num) bind(c)
-            use, intrinsic :: iso_c_binding, only: c_ptr, c_int
+            use :: iso_c_binding, only: c_ptr, c_int
             type(c_ptr), value :: device_ptr
             integer(c_int), value :: device_num
           end subroutine omp_target_free
 
           function omp_target_is_present(ptr, device_num) bind(c)
-            use, intrinsic :: iso_c_binding, only: c_ptr, c_int
+            use :: iso_c_binding, only: c_ptr, c_int
             integer(c_int) omp_target_is_present
             type(c_ptr), value :: ptr
             integer(c_int), value :: device_num
@@ -693,7 +693,7 @@
 
           function omp_target_memcpy(dst, src, length, dst_offset, src_offset, &
               dst_device_num, src_device_num) bind(c)
-            use, intrinsic :: iso_c_binding, only: c_ptr, c_size_t, c_int
+            use :: iso_c_binding, only: c_ptr, c_size_t, c_int
             integer(c_int) omp_target_memcpy
             type(c_ptr), value :: dst, src
             integer(c_size_t), value :: length, dst_offset, src_offset
@@ -703,7 +703,7 @@
           function omp_target_memcpy_rect(dst, src, element_size, num_dims,    &
               volume, dst_offsets, src_offsets, dst_dimensions,                &
               src_dimensions, dst_device_num, src_device_num) bind(c)
-            use, intrinsic :: iso_c_binding, only: c_ptr, c_size_t, c_int
+            use :: iso_c_binding, only: c_ptr, c_size_t, c_int
             integer(c_int) omp_target_memcpy_rect
             type(c_ptr), value :: dst, src
             integer(c_size_t), value :: element_size
@@ -716,7 +716,7 @@
               src_offset, dst_device_num, src_device_num, depobj_count,        &
               depobj_list) bind(c)
             use omp_lib_kinds
-            use, intrinsic :: iso_c_binding, only: c_ptr, c_size_t, c_int
+            use :: iso_c_binding, only: c_ptr, c_size_t, c_int
             integer(c_int) omp_target_memcpy_async
             type(c_ptr), value :: dst, src
             integer(c_size_t), value :: length, dst_offset, src_offset
@@ -730,7 +730,7 @@
               src_dimensions, dst_device_num, src_device_num, depobj_count,    &
               depobj_list) bind(c)
             use omp_lib_kinds
-            use, intrinsic :: iso_c_binding, only: c_ptr, c_size_t, c_int
+            use :: iso_c_binding, only: c_ptr, c_size_t, c_int
             integer(c_int) omp_target_memcpy_rect_async
             type(c_ptr), value :: dst, src
             integer(c_size_t), value :: element_size
@@ -742,7 +742,7 @@
           end function omp_target_memcpy_rect_async
 
           function omp_target_memset(ptr, val, count, device_num) bind(c)
-            use, intrinsic :: iso_c_binding, only : c_ptr, c_int, c_size_t
+            use :: iso_c_binding, only : c_ptr, c_int, c_size_t
             type(c_ptr) :: omp_target_memset
             type(c_ptr), value :: ptr
             integer(c_int), value :: val
@@ -753,7 +753,7 @@
           function omp_target_memset_async(ptr, val, count, device_num, &
                                            depobj_count, depobj_list) bind(c)
             use omp_lib_kinds
-            use, intrinsic :: iso_c_binding, only : c_ptr, c_int, c_size_t
+            use :: iso_c_binding, only : c_ptr, c_int, c_size_t
             type(c_ptr) :: omp_target_memset_async
             type(c_ptr), value :: ptr
             integer(c_int), value :: val
@@ -765,7 +765,7 @@
 
           function omp_target_associate_ptr(host_ptr, device_ptr, size,        &
               device_offset, device_num) bind(c)
-            use, intrinsic :: iso_c_binding, only: c_ptr, c_size_t, c_int
+            use :: iso_c_binding, only: c_ptr, c_size_t, c_int
             integer(c_int) omp_target_associate_ptr
             type(c_ptr), value :: host_ptr, device_ptr
             integer(c_size_t), value :: size, device_offset
@@ -773,21 +773,21 @@
           end function omp_target_associate_ptr
 
           function omp_get_mapped_ptr(ptr, device_num) bind(c)
-            use, intrinsic :: iso_c_binding, only: c_ptr, c_int
+            use :: iso_c_binding, only: c_ptr, c_int
             type(c_ptr) omp_get_mapped_ptr
             type(c_ptr), value :: ptr
             integer(c_int), value :: device_num
           end function omp_get_mapped_ptr
 
           function omp_target_disassociate_ptr(ptr, device_num) bind(c)
-            use, intrinsic :: iso_c_binding, only: c_ptr, c_int
+            use :: iso_c_binding, only: c_ptr, c_int
             integer(c_int) omp_target_disassociate_ptr
             type(c_ptr), value :: ptr
             integer(c_int), value :: device_num
           end function omp_target_disassociate_ptr
 
           function omp_target_is_accessible(ptr, size, device_num) bind(c)
-            use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t, c_int
+            use :: iso_c_binding, only : c_ptr, c_size_t, c_int
             integer(c_int) omp_target_is_accessible
             type(c_ptr), value :: ptr
             integer(c_size_t), value :: size
@@ -796,7 +796,7 @@
 
           function omp_alloc(size, allocator) bind(c)
             use omp_lib_kinds
-            use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t
+            use :: iso_c_binding, only : c_ptr, c_size_t
             type(c_ptr) omp_alloc
             integer(c_size_t), value :: size
             integer(omp_allocator_handle_kind), value :: allocator
@@ -804,7 +804,7 @@
 
           function omp_aligned_alloc(alignment, size, allocator) bind(c)
             use omp_lib_kinds
-            use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t
+            use :: iso_c_binding, only : c_ptr, c_size_t
             type(c_ptr) omp_aligned_alloc
             integer(c_size_t), value :: alignment, size
             integer(omp_allocator_handle_kind), value :: allocator
@@ -812,7 +812,7 @@
 
           function omp_calloc(nmemb, size, allocator) bind(c)
             use omp_lib_kinds
-            use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t
+            use :: iso_c_binding, only : c_ptr, c_size_t
             type(c_ptr) omp_calloc
             integer(c_size_t), value :: nmemb, size
             integer(omp_allocator_handle_kind), value :: allocator
@@ -820,7 +820,7 @@
 
           function omp_aligned_calloc(alignment, nmemb, size, allocator) bind(c)
             use omp_lib_kinds
-            use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t
+            use :: iso_c_binding, only : c_ptr, c_size_t
             type(c_ptr) omp_aligned_calloc
             integer(c_size_t), value :: alignment, nmemb, size
             integer(omp_allocator_handle_kind), value :: allocator
@@ -828,7 +828,7 @@
 
           function omp_realloc(ptr, size, allocator, free_allocator) bind(c)
             use omp_lib_kinds
-            use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t
+            use :: iso_c_binding, only : c_ptr, c_size_t
             type(c_ptr) omp_realloc
             type(c_ptr), value :: ptr
             integer(c_size_t), value :: size
@@ -838,7 +838,7 @@
 
           subroutine omp_free(ptr, allocator) bind(c)
             use omp_lib_kinds
-            use, intrinsic :: iso_c_binding, only : c_ptr
+            use :: iso_c_binding, only : c_ptr
             type(c_ptr), value :: ptr
             integer(omp_allocator_handle_kind), value :: allocator
           end subroutine omp_free
diff --git a/runtimes/cmake/config-Fortran.cmake b/runtimes/cmake/config-Fortran.cmake
index 74fd15021f0a0..6d2d3bdbf60ea 100644
--- a/runtimes/cmake/config-Fortran.cmake
+++ b/runtimes/cmake/config-Fortran.cmake
@@ -74,14 +74,8 @@ endfunction ()
 
 set(RUNTIMES_ENABLE_FORTRAN OFF)
 
-# Insert at least one element for
-#
-#    add_dependencies(target ${RUNTIMES_FORTRAN_BUILD_DEPS})
-#
-# to not fail
-add_custom_target(fortran-dummy-dep)
-set(RUNTIMES_FORTRAN_BUILD_DEPS fortran-dummy-dep)
 
+set(RUNTIMES_FORTRAN_BUILD_DEPS "")
 
 if (CMAKE_Fortran_COMPILER)
   # Workarounds for older versions of CMake not recognizing FLang. Hence, we
@@ -244,6 +238,22 @@ function (flang_module_target tgtname)
     "$<$<COMPILE_LANGUAGE:Fortran>:-fintrinsic-modules-path=${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}>"
   )
 
+  # Make CMake not ignore "use, intrinsic ::"-dependencies. Unfortunately,
+  # it is not universally handled by CMake s.t. we currently must not use
+  # "use, intrinsic ::" in our sources.
+  #  * CMake 3.22: Added Fortran_BUILDING_INSTRINSIC_MODULES handling "Unix Makefiles" generator
+  #  * CMake 4.0: Renamed INSTRINSIC to INTRINSIC (typo fix); INSTRINSIC spelling kept but deprecated
+  #  * CMake 4.5: Added handling by Ninja generators as well
+  set_target_properties(${tgtname}
+    PROPERTIES
+      Fortran_BUILDING_INTRINSIC_MODULES ON
+      Fortran_BUILDING_INSTRINSIC_MODULES ON
+  )
+
+  if (RUNTIMES_FORTRAN_BUILD_DEPS AND NOT tgtname IN_LIST RUNTIMES_FORTRAN_BUILD_DEPS)
+    target_link_libraries(${tgtname} PRIVATE ${RUNTIMES_FORTRAN_BUILD_DEPS})
+  endif ()
+
   if (CMAKE_Fortran_COMPILER_ID MATCHES "LLVM")
     target_compile_options(${tgtname} PRIVATE
       # Flang bug workaround: Reformating of cooked token buffer causes
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index e0f981d35b965..68bb1f1d60bf2 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -9175,6 +9175,23 @@ libc_support_library(
     ],
 )
 
+libc_support_library(
+    name = "__support_math_tanbf16",
+    hdrs = ["src/__support/math/tanbf16.h"],
+    deps = [
+        ":__support_fputil_bfloat16",
+        ":__support_fputil_cast",
+        ":__support_fputil_fenv_impl",
+        ":__support_fputil_fp_bits",
+        ":__support_fputil_multiply_add",
+        ":__support_macros_optimization",
+        ":__support_macros_properties_types",
+        ":__support_sincosf_utils",
+        ":hdr_errno_macros",
+        ":hdr_fenv_macros",
+    ],
+)
+
 libc_support_library(
     name = "__support_math_tanf",
     hdrs = ["src/__support/math/tanf.h"],
@@ -12882,6 +12899,13 @@ libc_math_function(
     additional_deps = [":__support_math_tan"],
 )
 
+libc_math_function(
+    name = "tanbf16",
+    additional_deps = [
+        ":__support_math_tanbf16",
+    ],
+)
+
 libc_math_function(
     name = "tanf",
     additional_deps = [":__support_math_tanf"],
diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index e746156841c2c..7de8be322a4b3 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -12,7 +12,7 @@ load("@rules_shell//shell:sh_binary.bzl", "sh_binary")
 load("//mlir:tblgen.bzl", "gentbl_cc_library", "gentbl_filegroup", "td_library")
 load(":binary_alias.bzl", "binary_alias")
 load(":config.bzl", "llvm_config_defines")
-load(":driver.bzl", "generate_driver_selects", "generate_driver_tools_def", "llvm_driver_cc_binary", "select_driver_tools")
+load(":driver.bzl", "generate_driver_selects", "generate_driver_tools_def", "llvm_driver_cc_binary")
 load(":enum_targets_gen.bzl", "enum_targets_gen")
 load(":targets.bzl", "llvm_targets")
 
@@ -821,13 +821,13 @@ cc_library(
 )
 
 # Command line flag to control which tools get included in the llvm driver binary.
-# The macro also generates config_setting targets used by select_driver_tools().
-generate_driver_selects(name = "driver-tools")
+# The macro also returns the selected tool dependencies.
+driver_tool_deps = generate_driver_selects(name = "driver-tools")
 
 generate_driver_tools_def(
     name = "gen_llvm_driver_tools_def",
     out = "LLVMDriverTools.def",
-    driver_tools = select_driver_tools(":driver-tools"),
+    driver_tools = ":driver-tools",
 )
 
 # Workaround inability to put `.def` files into `srcs` with a library
@@ -843,7 +843,7 @@ cc_binary(
     deps = [
         ":Support",
         ":llvm_driver_tools_def_lib",
-    ] + select_driver_tools(":driver-tools"),
+    ] + driver_tool_deps,
 )
 
 cc_binary(
diff --git a/utils/bazel/llvm-project-overlay/llvm/driver.bzl b/utils/bazel/llvm-project-overlay/llvm/driver.bzl
index 3ea0e6c4c59a6..3cbff1b1de5cf 100644
--- a/utils/bazel/llvm-project-overlay/llvm/driver.bzl
+++ b/utils/bazel/llvm-project-overlay/llvm/driver.bzl
@@ -6,7 +6,7 @@
 
 load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo")
 load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
-load("@rules_cc//cc:defs.bzl", "CcInfo", "cc_binary")
+load("@rules_cc//cc:defs.bzl", "cc_binary")
 
 # Mapping from every tool to the cc_library that implements the tool's entrypoint.
 _TOOLS = {
@@ -87,6 +87,8 @@ def generate_driver_selects(name):
 
     Args:
       name: the name of the flag that configures which tools are included.
+    Returns:
+      Selected driver tool dependencies.
     """
 
     _validated_string_list_flag(
@@ -94,43 +96,24 @@ def generate_driver_selects(name):
         build_setting_default = _TOOLS.keys(),
         values = _TOOLS.keys(),
     )
-    for tool in _TOOLS.keys():
+    driver_tool_deps = []
+    for tool, target in _TOOLS.items():
         native.config_setting(
             name = "{}-include-{}".format(name, tool),
             flag_values = {name: tool},
         )
-
-def select_driver_tools(flag):
-    """Produce a list of tool deps based on generate_driver_selects().
-
-    Args:
-      flag: name that was used for generate_driver_selects().
-    Returns:
-      List of tool deps based on generate_driver_selects().
-    """
-    tools = []
-    for tool, target in _TOOLS.items():
-        tools += select({
-            "{}-include-{}".format(flag, tool): [target],
+        driver_tool_deps += select({
+            "{}-include-{}".format(name, tool): [target],
             "//conditions:default": [],
         })
-    return tools
+    return driver_tool_deps
 
 def _generate_driver_tools_def_impl(ctx):
-    # Depending on how the LLVM build files are included,
-    # it may or may not have the @llvm-project repo prefix.
-    # Compare just on the name. We could also include the package,
-    # but the name itself is unique in practice.
-    label_to_name = {Label(v).name: k for k, v in _TOOLS.items()}
-
     # Reverse sort by the *main* tool name, but keep aliases together.
     # This is consistent with how tools/llvm-driver/CMakeLists.txt does it,
     # and this makes sure that more specific tools are checked first.
     # For example, "clang-scan-deps" should not match "clang".
-    tools = sorted(
-        [label_to_name[tool.label.name] for tool in ctx.attr.driver_tools],
-        reverse = True,
-    )
+    tools = sorted(ctx.attr.driver_tools[BuildSettingInfo].value, reverse = True)
     tool_alias_pairs = []
     for tool_name in tools:
         tool_alias_pairs.append((tool_name, tool_name))
@@ -156,9 +139,10 @@ generate_driver_tools_def = rule(
     doc = """Generate a list of LLVM_DRIVER_TOOL macros.
 See tools/llvm-driver/CMakeLists.txt for the reference implementation.""",
     attrs = {
-        "driver_tools": attr.label_list(
-            doc = "List of tools to include in the generated header. Use select_driver_tools() to provide this.",
-            providers = [CcInfo],
+        "driver_tools": attr.label(
+            doc = "Build setting created by generate_driver_selects().",
+            mandatory = True,
+            providers = [BuildSettingInfo],
         ),
         "out": attr.output(
             doc = "Name of the generated .def output file.",
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 490f624f9edea..21eea23a1a0ef 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -2137,6 +2137,7 @@ cc_library(
         ":IR",
         ":Pass",
         ":TransformUtils",
+        ":Transforms",
         ":VectorDialect",
     ],
 )
@@ -2361,6 +2362,7 @@ cc_library(
         ":MemRefDialect",
         ":Pass",
         ":TransformUtils",
+        ":Transforms",
         ":VectorDialect",
         "//llvm:Support",
     ],
@@ -4422,6 +4424,7 @@ cc_library(
         ":OpenACCToSCF",
         ":OpenMPToLLVM",
         ":PDLToPDLInterp",
+        ":RaiseWasmToMLIR",
         ":ReconcileUnrealizedCasts",
         ":SCFToControlFlow",
         ":SCFToEmitC",
@@ -8678,6 +8681,7 @@ cc_library(
         ":MathTransforms",
         ":Pass",
         ":TransformUtils",
+        ":Transforms",
         ":VectorDialect",
     ],
 )
@@ -8797,6 +8801,7 @@ cc_library(
         ":NVVMDialect",
         ":Pass",
         ":TransformUtils",
+        ":Transforms",
         ":VectorDialect",
         ":VectorUtils",
         "//llvm:Support",
@@ -11903,6 +11908,32 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "RaiseWasmToMLIR",
+    srcs = glob([
+        "lib/Conversion/RaiseWasm/*.cpp",
+    ]),
+    hdrs = glob([
+        "include/mlir/Conversion/RaiseWasm/*.h",
+    ]),
+    includes = ["include"],
+    deps = [
+        ":ArithDialect",
+        ":ControlFlowDialect",
+        ":ConversionPassIncGen",
+        ":FuncDialect",
+        ":IR",
+        ":MathDialect",
+        ":MemRefDialect",
+        ":Pass",
+        ":TransformUtils",
+        ":Transforms",
+        ":VectorDialect",
+        ":WasmSSADialect",
+        "//llvm:Support",
+    ],
+)
+
 cc_library(
     name = "VectorToLLVM",
     srcs = glob([
@@ -13332,6 +13363,7 @@ cc_library(
         ":MathDialect",
         ":Pass",
         ":TransformUtils",
+        ":Transforms",
         ":VectorDialect",
         "//llvm:Support",
     ],
diff --git a/utils/docs/llvm_sphinx/ext/checks.py b/utils/docs/llvm_sphinx/ext/checks.py
new file mode 100644
index 0000000000000..105cd571f44c9
--- /dev/null
+++ b/utils/docs/llvm_sphinx/ext/checks.py
@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+
+"""Sphinx extension for llvm-project checks/lints.
+
+Enable by adding "llvm_sphinx.checks" to the sphinx `extensions` list.
+"""
+
+import sys
+from typing import Dict, List
+from llvm_sphinx.help import venv_help
+
+try:
+    from sphinx.application import Sphinx
+    from sphinx.environment import BuildEnvironment
+    from sphinx.util import logging
+    from sphinx.errors import ExtensionError
+except ImportError as err:
+    print(venv_help(err), file=sys.stderr)
+    raise
+
+__version__ = "1.0"
+
+logger = logging.getLogger("llvm_sphinx.ext.checks")
+
+
+def setup(app: Sphinx) -> Dict[str, object]:
+    app.connect("env-updated", check_env)
+    return {
+        "version": __version__,
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }
+
+
+def check_env(app: Sphinx, env: BuildEnvironment) -> None:
+    toc_parents: Dict[str, List[str]] = {}
+    for parent, children in env.toctree_includes.items():
+        for child in children:
+            toc_parents.setdefault(child, []).append(parent)
+
+    for doc, parents in sorted(toc_parents.items()):
+        if len(parents) > 1:
+            # sphinx considers this an `info` only, and silently picks one
+            # parent arbitrarily. We upgrade this to an `error` to ensure
+            # we don't accumulate a growing list of offending toctrees.
+            logger.error(
+                "document is referenced in multiple toctrees: %s",
+                parents,
+                location=doc,
+            )