Merge pull request #5822 from ErnstPeng/la-dev

optimize zgemm, ic/zamin and sdot lsx kernel for 2k3000 cpu

Author: martin-frbg

driver/others/parameter.c

@@ -790,6 +790,17 @@ int get_L3_size() {
   return ((ret & 0xffff) + 1) * pow(2, ((ret >> 16) & 0xff)) * pow(2, ((ret >> 24) & 0x7f)) / 1024 / 1024; // MB
 }
 
+int get_cpu_prid() {
+  int ret = 0, id = 0x0;
+  __asm__ volatile (
+    "cpucfg %[ret], %[id]"
+    : [ret]"=r"(ret)
+    : [id]"r"(id)
+    : "memory"
+  );
+  return ret;
+}
+
 void blas_set_parameter(void){
 #if defined(LA464)
   int L3_size = get_L3_size();
@@ -868,6 +879,18 @@ void blas_set_parameter(void){
     }
   }
 #endif
+#elif defined(LA264)
+  int prid = get_cpu_prid();
+  if (prid == 0x0014b020) { //2k3000
+
+        zgemm_p = 128;
+        zgemm_q = 176;
+        zgemm_r = 360;
+  } else {
+        zgemm_p = 64;
+        zgemm_q = 120;
+        zgemm_r = 4096;
+  }
 #endif
 }
 #endif

kernel/loongarch64/KERNEL.LA264

@@ -1,6 +1,6 @@
 ifndef NO_LSX
 
-SDOTKERNEL  = dot_lsx.S
+SDOTKERNEL  = sdot_lsx.S
 DSDOTKERNEL = dot_lsx.S
 DDOTKERNEL  = dot_lsx.S
 CDOTKERNEL  = cdot_lsx.S

kernel/loongarch64/icamin_lsx.S

@@ -58,6 +58,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define VI3 $vr8
 #define VI4 $vr19
 #define VT0 $vr23
+#define VMASK $vr7
 
     PROLOGUE
     li.d  i0, 0
@@ -76,6 +77,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     li.d I, -1
     vreplgr2vr.d VI4, I
     vffint.d.l VI4, VI4 // -1
+    li.d I, 0x7fffffffffffffff  // Mask for clearing the sign bit
+    vreplgr2vr.d VMASK, I
     bne INCX, TEMP, .L20 // incx != 1
 
     // Init Index
@@ -99,17 +102,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     vld VX1, X, 2 * SIZE
     vpickev.d x1, VX1, VX0
     vpickod.d x2, VX1, VX0
-    vfmul.d x3, VI4, x1
-    vfmul.d x4, VI4, x2
-    vfcmp.clt.d VT0, x1, VI3
-    vfcmp.clt.d VINC8, x2, VI3
-    vbitsel.v x1, x1, x3, VT0
-    vbitsel.v x2, x2, x4, VINC8
+    vand.v  x1, x1, VMASK
+    vand.v  x2, x2, VMASK
     vfadd.d  VM0, x1, x2
 #else
     li.w I, -1
     vreplgr2vr.w VI4, I
     vffint.s.w VI4, VI4 // -1
+    li.d I, 0x7fffffff  // Mask for clearing the sign bit
+    vreplgr2vr.w VMASK, I
     bne INCX, TEMP, .L20 // incx != 1
 
     // Init Index
@@ -141,12 +142,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     vld VX1, X, 4 * SIZE
     vpickev.w x1, VX1, VX0
     vpickod.w x2, VX1, VX0
-    vfmul.s x3, VI4, x1
-    vfmul.s x4, VI4, x2
-    vfcmp.clt.s VT0, x1, VI3
-    vfcmp.clt.s VINC8, x2, VI3
-    vbitsel.v x1, x1, x3, VT0
-    vbitsel.v x2, x2, x4, VINC8
+    vand.v  x1, x1, VMASK
+    vand.v  x2, x2, VMASK
     vfadd.s  VM0, x1, x2
 #endif
     .align 3
@@ -159,12 +156,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     addi.d I, I, -1
     vpickev.d x1, VX1, VX0
     vpickod.d x2, VX1, VX0
-    vfmul.d x3, VI4, x1
-    vfmul.d x4, VI4, x2
-    vfcmp.clt.d VT0, x1, VI3
-    vfcmp.clt.d VINC8, x2, VI3
-    vbitsel.v x1, x1, x3, VT0
-    vbitsel.v x2, x2, x4, VINC8
+    vand.v  x1, x1, VMASK
+    vand.v  x2, x2, VMASK
     vfadd.d x1, x1, x2
     vfmin.d x3, VM0, x1
     vfcmp.ceq.d VT0, x3, VM0
@@ -183,12 +176,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     vpickev.w x1, VX1, VX0
     vpickod.w x2, VX1, VX0
 #endif
-    VFMUL  x3, VI4, x1
-    VFMUL  x4, VI4, x2
-    VCMPLT VT0, x1, VI3
-    VCMPLT VINC8, x2, VI3
-    vbitsel.v x1, x1, x3, VT0
-    vbitsel.v x2, x2, x4, VINC8
+    vand.v  x1, x1, VMASK
+    vand.v  x2, x2, VMASK
     VFADD  x1, x1, x2
     VFMIN  x3, VM0, x1
     VCMPEQ VT0, x3, VM0
@@ -264,12 +253,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     vinsgr2vr.d x2, t2, 0
     vinsgr2vr.d x1, t3, 1
     vinsgr2vr.d x2, t4, 1
-    vfmul.d x3, VI4, x1
-    vfmul.d x4, VI4, x2
-    vfcmp.clt.d VT0, x1, VI3
-    vfcmp.clt.d VINC8, x2, VI3
-    vbitsel.v x1, x1, x3, VT0
-    vbitsel.v x2, x2, x4, VINC8
+    vand.v  x1, x1, VMASK
+    vand.v  x2, x2, VMASK
     vfadd.d VM0, x1, x2
 #else
     addi.w i0, i0, 1
@@ -339,12 +324,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     vinsgr2vr.d x1, t3, 1
     vinsgr2vr.d x2, t4, 1
     vadd.d VI1, VI1, VINC4
-    vfmul.d x3, VI4, x1
-    vfmul.d x4, VI4, x2
-    vfcmp.clt.d VT0, x1, VI3
-    vfcmp.clt.d VINC8, x2, VI3
-    vbitsel.v x1, x1, x3, VT0
-    vbitsel.v x2, x2, x4, VINC8
+    vand.v  x1, x1, VMASK
+    vand.v  x2, x2, VMASK
     vfadd.d x1, x1, x2
     vfmin.d x3, VM0, x1
     ld.d t1, X, 0 * SIZE
@@ -385,12 +366,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     vinsgr2vr.w x2, t4, 3
 #endif
     addi.d I, I, -1
-    VFMUL  x3, VI4, x1
-    VFMUL  x4, VI4, x2
-    VCMPLT VT0, x1, VI3
-    VCMPLT VINC8, x2, VI3
-    vbitsel.v x1, x1, x3, VT0
-    vbitsel.v x2, x2, x4, VINC8
+    vand.v  x1, x1, VMASK
+    vand.v  x2, x2, VMASK
     VFADD  x1, x1, x2
     VFMIN  x3, VM0, x1
     VCMPEQ VT0, x3, VM0

kernel/loongarch64/sdot_lsx.S

@@ -0,0 +1,121 @@
+/***************************************************************************
+Copyright (c) 2023, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#define ASSEMBLER
+
+#include "common.h"
+
+#define N      $r4
+#define X      $r5
+#define INCX   $r6
+#define Y      $r7
+#define INCY   $r8
+
+#define I      $r17
+#define TEMP   $r18
+
+/* Don't change following FR unless you know the effects. */
+#define s1     $f8
+#define s2     $f9
+#define a1     $f10
+#define b1     $f11
+
+    PROLOGUE
+
+    vxor.v    $vr8,   $vr8,    $vr8
+    vxor.v    $vr9,   $vr9,    $vr9
+    slli.d    INCX,   INCX,    BASE_SHIFT
+    li.d      TEMP,   SIZE
+    slli.d    INCY,   INCY,    BASE_SHIFT
+    bge       $r0,    N,       .L999
+    bne       INCX,   TEMP,    .L20   /* inc_x=1 */
+    bne       INCY,   TEMP,    .L20   /* inc_y=1 */
+
+    /* ((inc_x == 1) && (inc_y == 1)) */
+    srai.d    I,      N,       4
+    bge       $r0,    I,       .L12   /* FLOAT: <16 */
+.L11:
+    /* FLOAT: 16~ */
+    vld       $vr0,   X,       0
+    vld       $vr1,   X,       16
+    vld       $vr2,   X,       32
+    vld       $vr3,   X,       48
+    vld       $vr4,   Y,       0
+    vld       $vr5,   Y,       16
+    vld       $vr6,   Y,       32
+    vld       $vr7,   Y,       48
+    addi.d    I,      I,       -1
+    addi.d    X,      X,       64
+    addi.d    Y,      Y,       64
+
+    vfmadd.s    $vr8,   $vr0,    $vr4,   $vr8
+    vfmadd.s    $vr9,   $vr1,    $vr5,   $vr9
+    vfmadd.s    $vr8,   $vr2,    $vr6,   $vr8
+    vfmadd.s    $vr9,   $vr3,    $vr7,   $vr9
+
+    bnez      I,      .L11
+
+    vfadd.s   $vr8,   $vr8,    $vr9
+    vextrins.w $vr1,  $vr8,    0x01
+    vextrins.w $vr2,  $vr8,    0x02
+    vextrins.w $vr3,  $vr8,    0x03
+    fadd.s     $f8,   $f8,     $f1
+    fadd.s     $f8,   $f8,     $f2
+    fadd.s     $f8,   $f8,     $f3
+.L12:
+    andi      I,      N,       0xf
+    bge       $r0,    I,       .L999
+.L13:
+    addi.d    I,      I,       -1
+    fld.s     $f0,    X,       0
+    fld.s     $f4,    Y,       0
+    addi.d    X,      X,       4
+    addi.d    Y,      Y,       4
+    fmadd.s   $f8,   $f0,   $f4,  $f8
+    bnez      I,      .L13
+    b         .L999
+
+    /* !((inc_x == 1) && (inc_y == 1)) */
+.L20:
+    move      I,      N
+.L21:
+    addi.d    I,      I,       -1
+
+    fld.s     $f0,    X,    0
+    fld.s     $f4,    Y,    0
+    add.d     X,      X,    INCX
+    add.d     Y,      Y,    INCY
+    fmadd.s   $f8,    $f0,  $f4,   $f8
+
+    bnez      I,      .L21
+    b         .L999
+
+.L999:
+    fmov.s    $f0,    $f8
+    jirl      $r0,    $r1,     0x0
+
+    EPILOGUE