SelectionDAGBuilder, mach-o: Skip trap after noreturn call (for Mach-O)

Add NoTrapAfterNoreturn target option which skips emission of traps
behind noreturn calls even if TrapUnreachable is enabled.

Enable the feature on Mach-O to save code size; Comments suggest it is
not possible to enable it for the other users of TrapUnreachable.

rdar://41530228

DifferentialRevision: https://reviews.llvm.org/D48674

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@335877 91177308-0d34-0410-b5e6-96231b3b80d8

Author: MatzeB

include/llvm/Target/TargetOptions.h

@@ -107,7 +107,9 @@ namespace llvm {
           EnableFastISel(false), UseInitArray(false),
           DisableIntegratedAS(false), RelaxELFRelocations(false),
           FunctionSections(false), DataSections(false),
-          UniqueSectionNames(true), TrapUnreachable(false), EmulatedTLS(false),
+          UniqueSectionNames(true), TrapUnreachable(false),
+          NoTrapAfterNoreturn(false),
+          EmulatedTLS(false),
           EnableIPRA(false), EmitStackSizeSection(false) {}
 
     /// PrintMachineCode - This flag is enabled when the -print-machineinstrs
@@ -209,6 +211,10 @@ namespace llvm {
     /// Emit target-specific trap instruction for 'unreachable' IR instructions.
     unsigned TrapUnreachable : 1;
 
+    /// Do not emit a trap instruction for 'unreachable' IR instructions behind
+    /// noreturn calls, even if TrapUnreachable is true.
+    unsigned NoTrapAfterNoreturn : 1;
+
     /// EmulatedTLS - This flag enables emulated TLS model, using emutls
     /// function in the runtime library..
     unsigned EmulatedTLS : 1;

lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

@@ -2584,9 +2584,23 @@ void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
 }
 
 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
-  if (DAG.getTarget().Options.TrapUnreachable)
-    DAG.setRoot(
-        DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
+  if (!DAG.getTarget().Options.TrapUnreachable)
+    return;
+
+  // We may be able to ignore unreachable behind a noreturn call.
+  if (DAG.getTarget().Options.NoTrapAfterNoreturn) {
+    const BasicBlock &BB = *I.getParent();
+    if (&I != &BB.front()) {
+      BasicBlock::const_iterator PredI =
+        std::prev(BasicBlock::const_iterator(&I));
+      if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
+        if (Call->doesNotReturn())
+          return;
+      }
+    }
+  }
+
+  DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
 }
 
 void SelectionDAGBuilder::visitFSub(const User &I) {

lib/Target/AArch64/AArch64TargetMachine.cpp

@@ -244,8 +244,10 @@ AArch64TargetMachine::AArch64TargetMachine(const Target &T, const Triple &TT,
       TLOF(createTLOF(getTargetTriple())), isLittle(LittleEndian) {
   initAsmInfo();
 
-  if (TT.isOSBinFormatMachO())
+  if (TT.isOSBinFormatMachO()) {
     this->Options.TrapUnreachable = true;
+    this->Options.NoTrapAfterNoreturn = true;
+  }
 }
 
 AArch64TargetMachine::~AArch64TargetMachine() = default;

lib/Target/ARM/ARMTargetMachine.cpp

@@ -238,8 +238,10 @@ ARMBaseTargetMachine::ARMBaseTargetMachine(const Target &T, const Triple &TT,
       this->Options.EABIVersion = EABI::EABI5;
   }
 
-  if (TT.isOSBinFormatMachO())
+  if (TT.isOSBinFormatMachO()) {
     this->Options.TrapUnreachable = true;
+    this->Options.NoTrapAfterNoreturn = true;
+  }
 
   initAsmInfo();
 }

lib/Target/X86/X86TargetMachine.cpp

@@ -221,8 +221,10 @@ X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT,
   // to ever want to mix 32 and 64-bit windows code in a single module
   // this should be fine.
   if ((TT.isOSWindows() && TT.getArch() == Triple::x86_64) || TT.isPS4() ||
-      TT.isOSBinFormatMachO())
+      TT.isOSBinFormatMachO()) {
     this->Options.TrapUnreachable = true;
+    this->Options.NoTrapAfterNoreturn = TT.isOSBinFormatMachO();
+  }
 
   initAsmInfo();
 }

test/CodeGen/X86/unreachable-trap.ll

@@ -0,0 +1,29 @@
+; RUN: llc -o - %s -mtriple=x86_64-windows-msvc | FileCheck %s --check-prefixes=CHECK,TRAP_AFTER_NORETURN
+; RUN: llc -o - %s -mtriple=x86_64-apple-darwin | FileCheck %s --check-prefixes=CHECK,NO_TRAP_AFTER_NORETURN
+
+; CHECK-LABEL: call_exit:
+; CHECK: callq {{_?}}exit
+; TRAP_AFTER_NORETURN: ud2
+; NO_TRAP_AFTER_NORETURN-NOT: ud2
+define i32 @call_exit() noreturn nounwind {
+  tail call void @exit(i32 0)
+  unreachable
+}
+
+; CHECK-LABEL: trap:
+; CHECK: ud2
+; TRAP_AFTER_NORETURN: ud2
+; NO_TRAP_AFTER_NORETURN-NOT: ud2
+define i32 @trap() noreturn nounwind {
+  tail call void @llvm.trap()
+  unreachable
+}
+
+; CHECK-LABEL: unreachable:
+; CHECK: ud2
+define i32 @unreachable() noreturn nounwind {
+  unreachable
+}
+
+declare void @llvm.trap() nounwind noreturn
+declare void @exit(i32 %rc) nounwind noreturn