[CodeGen] Merge ExpandLargeDivRem into ExpandFp (#172680)

Both passes expand instructions at the IR level.
They use the same kind of instruction visitation
logic and contain significant code duplication e.g.
for scalarization.

Author: frederik-h

llvm/docs/WritingAnLLVMPass.rst

@@ -673,7 +673,6 @@ default optimization pipelines, e.g. (the output has been trimmed):
   ModulePass Manager
   Pre-ISel Intrinsic Lowering
   FunctionPass Manager
-    Expand large div/rem
     Expand fp
     Expand Atomic instructions
   SVE intrinsics optimizations

llvm/include/llvm/CodeGen/ExpandLargeDivRem.h

@@ -546,9 +546,6 @@ LLVM_ABI FunctionPass *createExpandReductionsPass();
 // the corresponding function in a vector library (e.g., SVML, libmvec).
 LLVM_ABI FunctionPass *createReplaceWithVeclibLegacyPass();
 
-// Expands large div/rem instructions.
-LLVM_ABI FunctionPass *createExpandLargeDivRemPass();
-
 // Expands large div/rem instructions.
 LLVM_ABI FunctionPass *createExpandFpPass();
 

llvm/include/llvm/CodeGen/Passes.h

@@ -2219,7 +2219,7 @@ class LLVM_ABI TargetLoweringBase {
   }
 
   /// Returns the size in bits of the maximum div/rem the backend supports.
-  /// Larger operations will be expanded by ExpandLargeDivRem.
+  /// Larger operations will be expanded by ExpandFp.
   unsigned getMaxDivRemBitWidthSupported() const {
     return MaxDivRemBitWidthSupported;
   }
@@ -2885,7 +2885,7 @@ class LLVM_ABI TargetLoweringBase {
   }
 
   /// Set the size in bits of the maximum div/rem the backend supports.
-  /// Larger operations will be expanded by ExpandLargeDivRem.
+  /// Larger operations will be expanded by ExpandFp.
   void setMaxDivRemBitWidthSupported(unsigned SizeInBits) {
     MaxDivRemBitWidthSupported = SizeInBits;
   }
@@ -3742,7 +3742,7 @@ class LLVM_ABI TargetLoweringBase {
   unsigned MaxAtomicSizeInBitsSupported;
 
   /// Size in bits of the maximum div/rem size the backend supports.
-  /// Larger operations will be expanded by ExpandLargeDivRem.
+  /// Larger operations will be expanded by ExpandFp.
   unsigned MaxDivRemBitWidthSupported;
 
   /// Size in bits of the maximum fp to/from int conversion size the

llvm/include/llvm/CodeGen/TargetLowering.h

@@ -113,7 +113,6 @@ LLVM_ABI void initializeEarlyTailDuplicateLegacyPass(PassRegistry &);
 LLVM_ABI void initializeEdgeBundlesWrapperLegacyPass(PassRegistry &);
 LLVM_ABI void initializeEHContGuardTargetsPass(PassRegistry &);
 LLVM_ABI void initializeExpandFpLegacyPassPass(PassRegistry &);
-LLVM_ABI void initializeExpandLargeDivRemLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeExpandMemCmpLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeExpandPostRALegacyPass(PassRegistry &);
 LLVM_ABI void initializeExpandReductionsPass(PassRegistry &);

llvm/include/llvm/InitializePasses.h

@@ -128,7 +128,6 @@ struct ForcePassLinking {
     (void)llvm::createGVNPass();
     (void)llvm::createPostDomTree();
     (void)llvm::createMergeICmpsLegacyPass();
-    (void)llvm::createExpandLargeDivRemPass();
     (void)llvm::createExpandMemCmpLegacyPass();
     std::string buf;
     llvm::raw_string_ostream os(buf);

llvm/include/llvm/LinkAllPasses.h

@@ -33,7 +33,6 @@
 #include "llvm/CodeGen/DwarfEHPrepare.h"
 #include "llvm/CodeGen/EarlyIfConversion.h"
 #include "llvm/CodeGen/ExpandFp.h"
-#include "llvm/CodeGen/ExpandLargeDivRem.h"
 #include "llvm/CodeGen/ExpandMemCmp.h"
 #include "llvm/CodeGen/ExpandPostRAPseudos.h"
 #include "llvm/CodeGen/ExpandReductions.h"
@@ -678,7 +677,6 @@ void CodeGenPassBuilder<Derived, TargetMachineT>::addISelPasses(
     addModulePass(LowerEmuTLSPass(), PMW);
 
   addModulePass(PreISelIntrinsicLoweringPass(&TM), PMW);
-  addFunctionPass(ExpandLargeDivRemPass(TM), PMW);
   addFunctionPass(ExpandFpPass(TM, getOptLevel()), PMW);
 
   derived().addIRPasses(PMW);

llvm/include/llvm/Passes/CodeGenPassBuilder.h

@@ -57,7 +57,6 @@ add_llvm_component_library(LLVMCodeGen
   EdgeBundles.cpp
   EHContGuardTargets.cpp
   ExecutionDomainFix.cpp
-  ExpandLargeDivRem.cpp
   ExpandFp.cpp
   ExpandMemCmp.cpp
   ExpandPostRAPseudos.cpp

llvm/lib/CodeGen/CMakeLists.txt

@@ -39,7 +39,6 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeEarlyIfPredicatorPass(Registry);
   initializeEarlyMachineLICMPass(Registry);
   initializeEarlyTailDuplicateLegacyPass(Registry);
-  initializeExpandLargeDivRemLegacyPassPass(Registry);
   initializeExpandFpLegacyPassPass(Registry);
   initializeExpandMemCmpLegacyPassPass(Registry);
   initializeExpandPostRALegacyPass(Registry);

llvm/lib/CodeGen/CodeGen.cpp

@@ -12,6 +12,12 @@
 // useful for targets like x86_64 that cannot lower fp convertions
 // with more than 128 bits.
 //
+// This pass also expands div/rem instructions with a bitwidth above a
+// threshold into a call to auto-generated functions.  This is useful
+// for targets like x86_64 that cannot lower divisions with more than
+// 128 bits or targets like x86_32 that cannot lower divisions with
+// more than 64 bits.
+//
 //===----------------------------------------------------------------------===//
 
 #include "llvm/CodeGen/ExpandFp.h"
@@ -35,6 +41,8 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/IntegerDivision.h"
+#include <llvm/Support/Casting.h>
 #include <optional>
 
 #define DEBUG_TYPE "expand-fp"
@@ -47,7 +55,28 @@ static cl::opt<unsigned>
                         cl::desc("fp convert instructions on integers with "
                                  "more than <N> bits are expanded."));
 
+static cl::opt<unsigned>
+    ExpandDivRemBits("expand-div-rem-bits", cl::Hidden,
+                     cl::init(llvm::IntegerType::MAX_INT_BITS),
+                     cl::desc("div and rem instructions on integers with "
+                              "more than <N> bits are expanded."));
+
 namespace {
+bool isConstantPowerOfTwo(llvm::Value *V, bool SignedOp) {
+  auto *C = dyn_cast<ConstantInt>(V);
+  if (!C)
+    return false;
+
+  APInt Val = C->getValue();
+  if (SignedOp && Val.isNegative())
+    Val = -Val;
+  return Val.isPowerOf2();
+}
+
+bool isSigned(unsigned int Opcode) {
+  return Opcode == Instruction::SDiv || Opcode == Instruction::SRem;
+}
+
 /// This class implements a precise expansion of the frem instruction.
 /// The generated code is based on the fmod implementation in the AMD device
 /// libs.
@@ -995,11 +1024,17 @@ static bool runImpl(Function &F, const TargetLowering &TLI,
   if (ExpandFpConvertBits != llvm::IntegerType::MAX_INT_BITS)
     MaxLegalFpConvertBitWidth = ExpandFpConvertBits;
 
+  unsigned MaxLegalDivRemBitWidth = TLI.getMaxDivRemBitWidthSupported();
+  if (ExpandDivRemBits != llvm::IntegerType::MAX_INT_BITS)
+    MaxLegalDivRemBitWidth = ExpandDivRemBits;
+
   bool DisableExpandLargeFp =
       MaxLegalFpConvertBitWidth >= llvm::IntegerType::MAX_INT_BITS;
+  bool DisableExpandLargeDivRem =
+      MaxLegalDivRemBitWidth >= llvm::IntegerType::MAX_INT_BITS;
   bool DisableFrem = !FRemExpander::shouldExpandAnyFremType(TLI);
 
-  if (DisableExpandLargeFp && DisableFrem)
+  if (DisableExpandLargeFp && DisableFrem && DisableExpandLargeDivRem)
     return false;
 
   auto ShouldHandleInst = [&](Instruction &I) {
@@ -1021,6 +1056,16 @@ static bool runImpl(Function &F, const TargetLowering &TLI,
       return !DisableExpandLargeFp &&
              cast<IntegerType>(I.getOperand(0)->getType()->getScalarType())
                      ->getIntegerBitWidth() > MaxLegalFpConvertBitWidth;
+    case Instruction::UDiv:
+    case Instruction::SDiv:
+    case Instruction::URem:
+    case Instruction::SRem:
+      return !DisableExpandLargeDivRem &&
+             cast<IntegerType>(Ty->getScalarType())->getIntegerBitWidth() >
+                 MaxLegalDivRemBitWidth
+             // The backend has peephole optimizations for powers of two.
+             // TODO: We don't consider vectors here.
+             && !isConstantPowerOfTwo(I.getOperand(1), isSigned(I.getOpcode()));
     }
 
     return false;
@@ -1064,6 +1109,15 @@ static bool runImpl(Function &F, const TargetLowering &TLI,
     case Instruction::SIToFP:
       expandIToFP(I);
       break;
+
+    case Instruction::UDiv:
+    case Instruction::SDiv:
+      expandDivision(cast<BinaryOperator>(I));
+      break;
+    case Instruction::URem:
+    case Instruction::SRem:
+      expandRemainder(cast<BinaryOperator>(I));
+      break;
     }
   }