JIT: [experiment] inlining via tree splitting

src/coreclr/jit/async.cpp

@@ -156,9 +156,6 @@ PhaseStatus Compiler::SaveAsyncContexts()
             if (call->IsInlineCandidate() && (call->gtReturnType != TYP_VOID))
             {
                 restoreAfterStmt = stmt->GetNextStmt();
-                assert(restoreAfterStmt->GetRootNode()->OperIs(GT_RET_EXPR) ||
-                       (restoreAfterStmt->GetRootNode()->OperIs(GT_STORE_LCL_VAR) &&
-                        restoreAfterStmt->GetRootNode()->AsLclVarCommon()->Data()->OperIs(GT_RET_EXPR)));
             }
 
             if (curBB->hasTryIndex())

src/coreclr/jit/compiler.h

@@ -2542,6 +2542,7 @@ class Compiler
     friend class MorphCopyBlockHelper;
     friend class SharedTempsScope;
     friend class CallArgs;
+    friend class InlineAndDevirtualizeWalker;
     friend class IndirectCallTransformer;
     friend class ProfileSynthesis;
     friend class LocalsUseVisitor;
@@ -3455,7 +3456,6 @@ class Compiler
     GenTree* gtNewMustThrowException(unsigned helper, var_types type, CORINFO_CLASS_HANDLE clsHnd);
 
     GenTreeLclFld* gtNewLclFldNode(unsigned lnum, var_types type, unsigned offset, ClassLayout* layout = nullptr);
-    GenTreeRetExpr* gtNewInlineCandidateReturnExpr(GenTreeCall* inlineCandidate, var_types type);
 
     GenTreeFieldAddr* gtNewFieldAddrNode(var_types            type,
                                          CORINFO_FIELD_HANDLE fldHnd,
@@ -3667,7 +3667,7 @@ class Compiler
                                    bool         ignoreRoot       = false);
 
     bool gtSplitTree(
-        BasicBlock* block, Statement* stmt, GenTree* splitPoint, Statement** firstNewStmt, GenTree*** splitPointUse, bool early = false);
+        BasicBlock* block, Statement* stmt, GenTree* splitPoint, Statement** firstNewStmt, GenTree*** splitPointUse, bool includeOperands = true, bool early = false);
 
     bool gtStoreDefinesField(
         LclVarDsc* fieldVarDsc, ssize_t offset, unsigned size, ssize_t* pFieldStoreOffset, unsigned* pFieldStoreSize);
@@ -5136,7 +5136,7 @@ class Compiler
                            InlineCandidateInfo**  ppInlineCandidateInfo,
                            InlineResult*          inlineResult);
 
-    void impInlineRecordArgInfo(InlineInfo* pInlineInfo, CallArg* arg, InlArgInfo* argInfo, InlineResult* inlineResult);
+    void impInlineRecordArgInfo(InlineInfo* pInlineInfo, InlArgInfo* argInfo, CallArg* arg, InlineResult* inlineResult);
 
     void impInlineInitVars(InlineInfo* pInlineInfo);
 
@@ -5529,6 +5529,7 @@ class Compiler
     BasicBlock* fgSplitBlockAtBeginning(BasicBlock* curr);
     BasicBlock* fgSplitBlockAtEnd(BasicBlock* curr);
     BasicBlock* fgSplitBlockAfterStatement(BasicBlock* curr, Statement* stmt);
+    BasicBlock* fgSplitBlockBeforeStatement(BasicBlock* curr, Statement* stmt);
     BasicBlock* fgSplitBlockAfterNode(BasicBlock* curr, GenTree* node); // for LIR
     BasicBlock* fgSplitEdge(BasicBlock* curr, BasicBlock* succ);
     BasicBlock* fgSplitBlockBeforeTree(BasicBlock* block, Statement* stmt, GenTree* splitPoint, Statement** firstNewStmt, GenTree*** splitNodeUse);
@@ -6494,6 +6495,9 @@ class Compiler
 #endif
 
 public:
+    //                  Create a new temporary variable to hold the result of *ppTree,
+    //                  and transform the graph accordingly.
+    GenTree* fgInsertCommaFormTemp(GenTree** ppTree);
     Statement* fgNewStmtAtBeg(BasicBlock* block, GenTree* tree, const DebugInfo& di = DebugInfo());
     void fgInsertStmtAtEnd(BasicBlock* block, Statement* stmt);
     Statement* fgNewStmtAtEnd(BasicBlock* block, GenTree* tree, const DebugInfo& di = DebugInfo());
@@ -6505,9 +6509,9 @@ class Compiler
     void fgInsertStmtAfter(BasicBlock* block, Statement* insertionPoint, Statement* stmt);
     void fgInsertStmtBefore(BasicBlock* block, Statement* insertionPoint, Statement* stmt);
 
-    //                  Create a new temporary variable to hold the result of *ppTree,
-    //                  and transform the graph accordingly.
-    GenTree* fgInsertCommaFormTemp(GenTree** ppTree);
+private:
+    void fgInsertStmtListAtEnd(BasicBlock* block, Statement* stmtList);
+    void fgInsertStmtListBefore(BasicBlock* block, Statement* stmtBefore, Statement* stmtList);
 
 private:
     Statement* fgInsertStmtListAfter(BasicBlock* block, Statement* stmtAfter, Statement* stmtList);
@@ -6648,8 +6652,8 @@ class Compiler
     GenTree* fgMorphCall(GenTreeCall* call);
     GenTree* fgExpandVirtualVtableCallTarget(GenTreeCall* call);
 
-    void fgMorphCallInline(GenTreeCall* call, InlineResult* result);
-    void fgMorphCallInlineHelper(GenTreeCall* call, InlineResult* result, InlineContext** createdContext);
+    void fgMorphCallInline(InlineInfo& inlineInfo, GenTreeCall* call, InlineResult* result);
+    void fgMorphCallInlineHelper(InlineInfo& inlineInfo, GenTreeCall* call, InlineResult* result, InlineContext** createdContext);
 #if DEBUG
     void fgNoteNonInlineCandidate(Statement* stmt, GenTreeCall* call);
     static fgWalkPreFn fgFindNonInlineCandidate;
@@ -6829,11 +6833,11 @@ class Compiler
     bool MethodInstantiationComplexityExceeds(CORINFO_METHOD_HANDLE handle, int& cur, int max);
     bool TypeInstantiationComplexityExceeds(CORINFO_CLASS_HANDLE handle, int& cur, int max);
 
-    void fgInvokeInlineeCompiler(GenTreeCall* call, InlineResult* result, InlineContext** createdContext);
-    void fgInsertInlineeBlocks(InlineInfo* pInlineInfo);
-    void fgInsertInlineeArgument(const InlArgInfo& argInfo, BasicBlock* block, Statement** afterStmt, Statement** newStmt, const DebugInfo& callDI);
-    Statement* fgInlinePrependStatements(InlineInfo* inlineInfo);
-    void fgInlineAppendStatements(InlineInfo* inlineInfo, BasicBlock* block, Statement* stmt);
+    void fgInvokeInlineeCompiler(InlineInfo& inlineInfo, GenTreeCall* call, InlineResult* result, InlineContext** createdContext);
+    void fgFinalizeInlineeStatements(InlineInfo* pInlineInfo);
+    void fgInsertInlineeArgument(class StatementListBuilder& statements, const InlArgInfo& argInfo, const DebugInfo& callDI);
+    void fgInlinePrependStatements(InlineInfo* inlineInfo);
+    void fgInlineAppendStatements(class StatementListBuilder& statements, InlineInfo* inlineInfo);
 
 #ifdef DEBUG
     static fgWalkPreFn fgDebugCheckInlineCandidates;
@@ -11698,7 +11702,6 @@ class GenTreeVisitor
             case GT_ASYNC_CONTINUATION:
             case GT_LABEL:
             case GT_FTN_ADDR:
-            case GT_RET_EXPR:
             case GT_CNS_INT:
             case GT_CNS_LNG:
             case GT_CNS_DBL:

src/coreclr/jit/compiler.hpp

@@ -4493,7 +4493,6 @@ GenTree::VisitResult GenTree::VisitOperands(TVisitor visitor)
         case GT_ASYNC_CONTINUATION:
         case GT_LABEL:
         case GT_FTN_ADDR:
-        case GT_RET_EXPR:
         case GT_CNS_INT:
         case GT_CNS_LNG:
         case GT_CNS_DBL:

src/coreclr/jit/debuginfo.cpp

@@ -113,17 +113,17 @@ void DebugInfo::Validate() const
         if (!di.IsValid())
             continue;
 
-        bool isValidOffs = di.GetLocation().GetOffset() < di.GetInlineContext()->GetILSize();
-        if (isValidOffs)
-        {
-            bool isValidStart = di.GetInlineContext()->GetILInstsSet()->bitVectTest(di.GetLocation().GetOffset());
-            assert(isValidStart &&
-                   "Detected invalid debug info: IL offset does not refer to the start of an IL instruction");
-        }
-        else
-        {
-            assert(!"Detected invalid debug info: IL offset is out of range");
-        }
+        // bool isValidOffs = di.GetLocation().GetOffset() < di.GetInlineContext()->GetILSize();
+        // if (isValidOffs)
+        //{
+        //     bool isValidStart = di.GetInlineContext()->GetILInstsSet()->bitVectTest(di.GetLocation().GetOffset());
+        //     assert(isValidStart &&
+        //            "Detected invalid debug info: IL offset does not refer to the start of an IL instruction");
+        // }
+        // else
+        //{
+        //     assert(!"Detected invalid debug info: IL offset is out of range");
+        // }
 
     } while (di.GetParent(&di));
 }

src/coreclr/jit/fgbasic.cpp

@@ -4743,6 +4743,16 @@ BasicBlock* Compiler::fgSplitBlockAfterStatement(BasicBlock* curr, Statement* st
     return newBlock;
 }
 
+BasicBlock* Compiler::fgSplitBlockBeforeStatement(BasicBlock* curr, Statement* stmt)
+{
+    if (stmt == curr->firstStmt())
+    {
+        return fgSplitBlockAtBeginning(curr);
+    }
+
+    return fgSplitBlockAfterStatement(curr, stmt->GetPrevStmt());
+}
+
 //------------------------------------------------------------------------------
 // fgSplitBlockBeforeTree : Split the given block right before the given tree
 //