Replace successive "ldr" and "str" instructions with "ldp" and "stp"

src/coreclr/jit/emitarm64.cpp

@@ -5676,6 +5676,14 @@ void emitter::emitIns_R_R_I(
         {
             return;
         }
+
+        // If we have replaced an LDR or STR instruction with
+        // an LDP or STP then we do not want to carry on to
+        // emit the second instruction.
+        if (ReplacedLdrStr(ins, attr, reg1, reg2, imm, size, fmt))
+        {
+            return;
+        }
     }
     else if (isAddSub)
     {
@@ -6491,7 +6499,8 @@ void emitter::emitIns_R_R_R_I(instruction ins,
                               regNumber   reg3,
                               ssize_t     imm,
                               insOpts     opt /* = INS_OPTS_NONE */,
-                              emitAttr    attrReg2 /* = EA_UNKNOWN */)
+                              emitAttr    attrReg2 /* = EA_UNKNOWN */,
+                              instrDesc*  reuseInstr /* = nullptr */)
 {
     emitAttr  size     = EA_SIZE(attr);
     emitAttr  elemsize = EA_UNKNOWN;
@@ -6626,6 +6635,7 @@ void emitter::emitIns_R_R_R_I(instruction ins,
                 scale = (size == EA_8BYTE) ? 3 : 2;
             }
             isLdSt = true;
+            fmt    = IF_LS_3C;
             break;
 
         case INS_ld1:
@@ -6906,7 +6916,52 @@ void emitter::emitIns_R_R_R_I(instruction ins,
     }
     assert(fmt != IF_NONE);
 
-    instrDesc* id = emitNewInstrCns(attr, imm);
+    // An "instrDesc" will *always* be required.
+    // Under normal circumstances the instruction
+    // will be added to the emitted group. However,
+    // this is not correct for instructions that
+    // are going to overwrite already-emitted
+    // instructions.
+    instrDesc* id;
+    INDEBUG(size_t reusedInstrSize = (reuseInstr != nullptr) ? emitSizeOfInsDsc(reuseInstr) : 0);
+
+    // Now the instruction is either emitted OR
+    // used to overwrite the previously-emitted
+    // instruction.
+    if (reuseInstr == nullptr)
+    {
+        id = emitNewInstrCns(attr, imm);
+    }
+    else
+    {
+        id = reuseInstr;
+        memset(id, 0, sizeof(instrDesc));
+
+        // Store the size and handle the two special
+        // values that indicate GCref and ByRef
+
+        if (EA_IS_GCREF(attr))
+        {
+            // A special value indicates a GCref pointer value
+
+            id->idGCref(GCT_GCREF);
+            id->idOpSize(EA_PTRSIZE);
+        }
+        else if (EA_IS_BYREF(attr))
+        {
+            // A special value indicates a Byref pointer value
+
+            id->idGCref(GCT_BYREF);
+            id->idOpSize(EA_PTRSIZE);
+        }
+        else
+        {
+            id->idGCref(GCT_NONE);
+            id->idOpSize(EA_SIZE(attr));
+        }
+
+        id->idSmallCns(imm);
+    }
 
     id->idIns(ins);
     id->idInsFmt(fmt);
@@ -6932,8 +6987,15 @@ void emitter::emitIns_R_R_R_I(instruction ins,
         }
     }
 
-    dispIns(id);
-    appendToCurIG(id);
+    assert((reuseInstr == nullptr) || (emitSizeOfInsDsc(reuseInstr) == reusedInstrSize));
+
+    // Now the instruction is EITHER emitted OR used to overwrite the previously-emitted instruction.
+    if (reuseInstr == nullptr)
+    {
+        // Then this is the standard exit path and the instruction is to be appended to the instruction group.
+        dispIns(id);
+        appendToCurIG(id);
+    }
 }
 
 /*****************************************************************************
@@ -7623,8 +7685,7 @@ void emitter::emitIns_R_S(instruction ins, emitAttr attr, regNumber reg1, int va
     {
         bool    useRegForImm = false;
         ssize_t mask         = (1 << scale) - 1; // the mask of low bits that must be zero to encode the immediate
-
-        imm = disp;
+        imm                  = disp;
         if (imm == 0)
         {
             fmt = IF_LS_2A;
@@ -7670,6 +7731,18 @@ void emitter::emitIns_R_S(instruction ins, emitAttr attr, regNumber reg1, int va
 
     assert(fmt != IF_NONE);
 
+    // This handles LDR duplicate instructions
+
+    // If we have replaced an LDR or STR instruction with
+    // an LDP or STP then we do not want to carry on to
+    // emit the second instruction.
+    if (ReplacedLdrStr(ins, attr, reg1, reg2, imm, size, fmt))
+    {
+        return;
+    }
+
+    // We need to simply emit the instruction unchanged
+
     instrDesc* id = emitNewInstrCns(attr, imm);
 
     id->idIns(ins);
@@ -7901,6 +7974,14 @@ void emitter::emitIns_S_R(instruction ins, emitAttr attr, regNumber reg1, int va
 
     assert(fmt != IF_NONE);
 
+    // If we have replaced an LDR or STR instruction with
+    // an LDP or STP then we do not want to carry on to
+    // emit the second instruction.
+    if (ReplacedLdrStr(ins, attr, reg1, reg2, imm, size, fmt))
+    {
+        return;
+    }
+
     instrDesc* id = emitNewInstrCns(attr, imm);
 
     id->idIns(ins);
@@ -16128,4 +16209,181 @@ bool emitter::IsRedundantLdStr(
 
     return false;
 }
+
+//-----------------------------------------------------------------------------------
+// ReplacedLdrStr: Potentially, overwrite a previously-emitted
+//                 "ldr" or "str" instruction with an "ldp" or
+//                 "stp" instruction.
+//
+// Arguments:
+//     ins      - The instruction code
+//     reg1Attr - The emit attribute for register 1
+//     reg1     - Register 1 number
+//     reg2     - Register 2 number
+//     imm      - Immediate offset, prior to scaling by operand size
+//     size     - Operand size
+//     fmt      - Instruction format
+//
+// Return Value:
+//    "true" if the previous instruction HAS been overwritten.
+
+bool emitter::ReplacedLdrStr(
+    instruction ins, emitAttr reg1Attr, regNumber reg1, regNumber reg2, ssize_t imm, emitAttr size, insFormat fmt)
+{
+    if (emitComp->opts.OptimizationEnabled())
+    {
+        RegisterOrder optimizationOrder = IsOptimizableLdrStr(ins, reg1, reg2, imm, size, fmt);
+
+        if (optimizationOrder != eRO_none)
+        {
+            regNumber oldReg1 = emitLastIns->idReg1();
+            ssize_t   oldImm =
+                emitLastIns->idIsLargeCns() ? ((instrDescCns*)emitLastIns)->idcCnsVal : emitLastIns->idSmallCns();
+            instruction optIns = (ins == INS_ldr) ? INS_ldp : INS_stp;
+
+            emitAttr oldReg1Attr;
+            switch (emitLastIns->idGCref())
+            {
+                case GCT_GCREF:
+                    oldReg1Attr = EA_GCREF;
+                    break;
+                case GCT_BYREF:
+                    oldReg1Attr = EA_BYREF;
+                    break;
+                default:
+                    oldReg1Attr = emitLastIns->idOpSize();
+                    break;
+            }
+
+            // Overwrite the "sub-optimal" instruction with the *optimised* instruction, directly
+            // into the output buffer.
+            if (optimizationOrder == eRO_ascending)
+            {
+                // The FIRST register is at the lower offset
+                emitIns_R_R_R_I(optIns, oldReg1Attr, oldReg1, reg1, reg2, oldImm * size, INS_OPTS_NONE, reg1Attr,
+                                emitLastIns);
+            }
+            else
+            {
+                // The SECOND register is at the lower offset
+                emitIns_R_R_R_I(optIns, reg1Attr, reg1, oldReg1, reg2, imm * size, INS_OPTS_NONE, oldReg1Attr,
+                                emitLastIns);
+            }
+
+            // And now return true, to indicate that the second instruction descriptor is no longer to be emitted.
+            return true;
+        }
+    }
+
+    return false;
+}
+
+//-----------------------------------------------------------------------------------
+// IsOptimizableLdrStr: Check if it is possible to optimize two "ldr" or "str"
+//                      instructions into a single "ldp" or "stp" instruction.
+//
+// Arguments:
+//     ins  - The instruction code
+//     reg1 - Register 1 number
+//     reg2 - Register 2 number
+//     imm  - Immediate offset, prior to scaling by operand size
+//     size - Operand size
+//     fmt  - Instruction format
+//
+// Return Value:
+//    eRO_none       - No optimization of consecutive instructions is possible
+//    eRO_ascending  - Registers can be loaded/ stored into ascending store locations
+//    eRO_descending - Registers can be loaded/ stored into decending store locations.
+
+emitter::RegisterOrder emitter::IsOptimizableLdrStr(
+    instruction ins, regNumber reg1, regNumber reg2, ssize_t imm, emitAttr size, insFormat fmt)
+{
+    bool isFirstInstrInBlock = (emitCurIGinsCnt == 0) && ((emitCurIG->igFlags & IGF_EXTEND) == 0);
+
+    RegisterOrder optimisationOrder = eRO_none;
+
+    if (((ins != INS_ldr) && (ins != INS_str)) || (isFirstInstrInBlock) || (emitLastIns == nullptr))
+    {
+        return eRO_none;
+    }
+
+    if (ins != emitLastIns->idIns())
+    {
+        // Not successive ldr or str instructions
+        return eRO_none;
+    }
+
+    if (emitSizeOfInsDsc(emitLastIns) != sizeof(instrDesc))
+    {
+        // Not instruction descriptors of the same, standard size.
+        return eRO_none;
+    }
+
+    regNumber prevReg1   = emitLastIns->idReg1();
+    regNumber prevReg2   = emitLastIns->idReg2();
+    insFormat lastInsFmt = emitLastIns->idInsFmt();
+    emitAttr  prevSize   = emitLastIns->idOpSize();
+    ssize_t prevImm = emitLastIns->idIsLargeCns() ? ((instrDescCns*)emitLastIns)->idcCnsVal : emitLastIns->idSmallCns();
+
+    // Signed, *raw* immediate value fits in 7 bits, so for LDP/ STP the raw value is from -64 to +63.
+    // For LDR/ STR, there are 9 bits, so we need to limit the range explicitly in software.
+    if ((imm < -64) || (imm > 63) || (prevImm < -64) || (prevImm > 63))
+    {
+        // Then one or more of the immediate values is out of range, so we cannot optimise.
+        return eRO_none;
+    }
+
+    if ((!isGeneralRegisterOrZR(reg1)) || (!isGeneralRegisterOrZR(prevReg1)))
+    {
+        // Either register 1 is not a general register or previous register 1 is not a general register
+        // or the zero register, so we cannot optimise.
+        return eRO_none;
+    }
+
+    if (lastInsFmt != fmt)
+    {
+        // The formats of the two instructions differ.
+        return eRO_none;
+    }
+
+    if ((emitInsIsLoad(ins)) && (reg1 == prevReg1))
+    {
+        // Cannot load to the same register twice.
+        return eRO_none;
+    }
+
+    if (prevSize != size)
+    {
+        // Operand sizes differ.
+        return eRO_none;
+    }
+
+    // There are two possible orders for consecutive registers.
+    // These may be stored to or loaded from increasing or
+    // decreasing store locations.
+    if (imm == (prevImm + 1))
+    {
+        // Previous Register 1 is at a higher offset than This Register 1
+        optimisationOrder = eRO_ascending;
+    }
+    else if (imm == (prevImm - 1))
+    {
+        // Previous Register 1 is at a loker offset than This Register 1
+        optimisationOrder = eRO_descending;
+    }
+    else
+    {
+        // Not consecutive immediate values.
+        return eRO_none;
+    }
+
+    if ((reg2 != prevReg2) || !isGeneralRegisterOrSP(reg2))
+    {
+        // The "register 2" should be same as previous instruction and should either be a general register or stack
+        // pointer.
+        return eRO_none;
+    }
+    return optimisationOrder;
+}
+
 #endif // defined(TARGET_ARM64)

src/coreclr/jit/emitarm64.h

@@ -69,6 +69,17 @@ instrDesc* emitNewInstrCallInd(int              argCnt,
                                emitAttr         retSize,
                                emitAttr         secondRetSize);
 
+/************************************************************************/
+/*   enum to allow instruction optimisation to specify register order   */
+/************************************************************************/
+
+enum RegisterOrder
+{
+    eRO_none = 0,
+    eRO_ascending,
+    eRO_descending
+};
+
 /************************************************************************/
 /*               Private helpers for instruction output                 */
 /************************************************************************/
@@ -112,7 +123,10 @@ static UINT64 Replicate_helper(UINT64 value, unsigned width, emitAttr size);
 static bool IsMovInstruction(instruction ins);
 bool IsRedundantMov(instruction ins, emitAttr size, regNumber dst, regNumber src, bool canSkip);
 bool IsRedundantLdStr(instruction ins, regNumber reg1, regNumber reg2, ssize_t imm, emitAttr size, insFormat fmt);
-
+bool ReplacedLdrStr(
+    instruction ins, emitAttr reg1Attr, regNumber reg1, regNumber reg2, ssize_t imm, emitAttr size, insFormat fmt);
+RegisterOrder IsOptimizableLdrStr(
+    instruction ins, regNumber reg1, regNumber reg2, ssize_t imm, emitAttr size, insFormat fmt);
 /************************************************************************
 *
 * This union is used to to encode/decode the special ARM64 immediate values
@@ -775,8 +789,9 @@ void emitIns_R_R_R_I(instruction ins,
                      regNumber   reg2,
                      regNumber   reg3,
                      ssize_t     imm,
-                     insOpts     opt      = INS_OPTS_NONE,
-                     emitAttr    attrReg2 = EA_UNKNOWN);
+                     insOpts     opt        = INS_OPTS_NONE,
+                     emitAttr    attrReg2   = EA_UNKNOWN,
+                     instrDesc*  reuseInstr = nullptr);
 
 void emitIns_R_R_R_Ext(instruction ins,
                        emitAttr    attr,