Fix 32-bit integer overflow in LAPACKE nancheck index arithmetic

The optional NaN checks performed by LAPACKE front-ends computed array
offsets and lengths in 32-bit lapack_int arithmetic, which overflows for
large matrices (e.g. LAPACKE_dpotrf with n = lda >= 46341 overflows
j*lda in LAPACKE_dtr_nancheck, leading to out-of-bounds reads and
crashes). Several nancheck helpers (ge, gb, tp, tf) already cast to
size_t; this applies the same treatment to the ones that were missed:

- ?tr_nancheck: cast a[i+j*lda] index to size_t (also covers po/sy/he
  checks, i.e. the POTRF/Cholesky path)
- ?_nancheck (vector): iterate by element count instead of n*inc, with
  size_t indexing
- ?sp/?pp/?pf/?hp/?tp/?tf_nancheck: compute n*(n+1)/2 length in size_t
- ?tz_nancheck: compute triangular/rectangular part offsets in size_t,
  replacing the -1 sentinel with an explicit flag

Verified with UBSan: LAPACKE_str_nancheck at n = lda = 47000 previously
reported signed integer overflow and crashed with SEGV at -O0; it now
passes and still detects the NaN. Old and new code produce identical
results across 1907 enumerated small-size cases (all layouts, uplo,
diag, shapes and NaN positions for tr/tz/vector/packed variants).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: changangela

lapack-netlib/LAPACKE/utils/lapacke_c_nancheck.c

@@ -42,8 +42,8 @@ lapack_logical LAPACKE_c_nancheck( lapack_int n,
     if( incx == 0 ) return (lapack_logical) LAPACK_CISNAN( x[0] );
     inc = ( incx > 0 ) ? incx : -incx ;
 
-    for( i = 0; i < n*inc; i+=inc ) {
-        if( LAPACK_CISNAN( x[i] ) )
+    for( i = 0; i < n; i++ ) {
+        if( LAPACK_CISNAN( x[(size_t)i*inc] ) )
             return (lapack_logical) 1;
     }
     return (lapack_logical) 0;

lapack-netlib/LAPACKE/utils/lapacke_chp_nancheck.c

@@ -39,6 +39,6 @@
 lapack_logical LAPACKE_chp_nancheck( lapack_int n,
                                       const lapack_complex_float *ap )
 {
-    lapack_int len = n*(n+1)/2;
+    lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
     return LAPACKE_c_nancheck( len, ap, 1 );
 }

lapack-netlib/LAPACKE/utils/lapacke_cpf_nancheck.c

@@ -40,6 +40,6 @@
 lapack_logical LAPACKE_cpf_nancheck( lapack_int n,
                                       const lapack_complex_float *a )
 {
-    lapack_int len = n*(n+1)/2;
+    lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
     return LAPACKE_c_nancheck( len, a, 1 );
 }

lapack-netlib/LAPACKE/utils/lapacke_cpp_nancheck.c

@@ -39,6 +39,6 @@
 lapack_logical LAPACKE_cpp_nancheck( lapack_int n,
                                       const lapack_complex_float *ap )
 {
-    lapack_int len = n*(n+1)/2;
+    lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
     return LAPACKE_c_nancheck( len, ap, 1 );
 }

lapack-netlib/LAPACKE/utils/lapacke_csp_nancheck.c

@@ -39,6 +39,6 @@
 lapack_logical LAPACKE_csp_nancheck( lapack_int n,
                                       const lapack_complex_float *ap )
 {
-    lapack_int len = n*(n+1)/2;
+    lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
     return LAPACKE_c_nancheck( len, ap, 1 );
 }

lapack-netlib/LAPACKE/utils/lapacke_ctf_nancheck.c

@@ -150,7 +150,7 @@ lapack_logical LAPACKE_ctf_nancheck( int matrix_layout, char transr,
         }
     } else {
         /* Non-unit case - just check whole array for NaNs. */
-        len = n*(n+1)/2;
+        len = (lapack_int)(((size_t)n*(n+1))/2);
         return LAPACKE_cge_nancheck( LAPACK_COL_MAJOR, len, 1, a, len );
     }
 }

lapack-netlib/LAPACKE/utils/lapacke_ctp_nancheck.c

@@ -76,7 +76,7 @@ lapack_logical LAPACKE_ctp_nancheck( int matrix_layout, char uplo, char diag,
         return (lapack_logical) 0;
     } else {
         /* Non-unit case - just check whole array for NaNs. */
-        len = n*(n+1)/2;
+        len = (lapack_int)(((size_t)n*(n+1))/2);
         return LAPACKE_c_nancheck( len, ap, 1 );
     }
 }

lapack-netlib/LAPACKE/utils/lapacke_ctr_nancheck.c

@@ -68,14 +68,14 @@ lapack_logical LAPACKE_ctr_nancheck( int matrix_layout, char uplo, char diag,
     if( ( colmaj || lower ) && !( colmaj && lower ) ) {
         for( j = st; j < n; j++ ) {
             for( i = 0; i < MIN( j+1-st, lda ); i++ ) {
-                if( LAPACK_CISNAN( a[i+j*lda] ) )
+                if( LAPACK_CISNAN( a[i+(size_t)j*lda] ) )
                     return (lapack_logical) 1;
             }
         }
     } else {
         for( j = 0; j < n-st; j++ ) {
             for( i = j+st; i < MIN( n, lda ); i++ ) {
-                if( LAPACK_CISNAN( a[i+j*lda] ) )
+                if( LAPACK_CISNAN( a[i+(size_t)j*lda] ) )
                     return (lapack_logical) 1;
             }
         }

lapack-netlib/LAPACKE/utils/lapacke_ctz_nancheck.c

@@ -103,35 +103,40 @@ lapack_logical LAPACKE_ctz_nancheck( int matrix_layout, char direct, char uplo,
     }
 
     /* Initial offsets and sizes of triangular and rectangular parts */
-    lapack_int tri_offset = 0;
+    size_t tri_offset = 0;
     lapack_int tri_n = MIN(m,n);
-    lapack_int rect_offset = -1;
+    size_t rect_offset = 0;
+    lapack_logical check_rect = 0;
     lapack_int rect_m = ( m > n ) ? m - n : m;
     lapack_int rect_n = ( n > m ) ? n - m : n;
 
     /* Fix offsets depending on the shape of the matrix */
     if( front ) {
         if( lower && m > n ) {
-            rect_offset = tri_n * ( !colmaj ? lda : 1 );
+            rect_offset = (size_t)tri_n * ( !colmaj ? lda : 1 );
+            check_rect = 1;
         } else if( !lower && n > m ) {
-            rect_offset = tri_n * ( colmaj ? lda : 1 );
+            rect_offset = (size_t)tri_n * ( colmaj ? lda : 1 );
+            check_rect = 1;
         }
     } else {
         if( m > n ) {
-            tri_offset = rect_m * ( !colmaj ? lda : 1 );
+            tri_offset = (size_t)rect_m * ( !colmaj ? lda : 1 );
             if( !lower ) {
                 rect_offset = 0;
+                check_rect = 1;
             }
         } else if( n > m ) {
-            tri_offset = rect_n * ( colmaj ? lda : 1 );
+            tri_offset = (size_t)rect_n * ( colmaj ? lda : 1 );
             if( lower ) {
                 rect_offset = 0;
+                check_rect = 1;
             }
         }
     }
 
     /* Check rectangular part */
-    if( rect_offset >= 0 ) {
+    if( check_rect ) {
         if( LAPACKE_cge_nancheck( matrix_layout, rect_m, rect_n,
                                   &a[rect_offset], lda) ) {
             return (lapack_logical) 1;

lapack-netlib/LAPACKE/utils/lapacke_d_nancheck.c

@@ -42,8 +42,8 @@ lapack_logical LAPACKE_d_nancheck( lapack_int n,
     if( incx == 0 ) return (lapack_logical) LAPACK_DISNAN( x[0] );
     inc = ( incx > 0 ) ? incx : -incx ;
 
-    for( i = 0; i < n*inc; i+=inc ) {
-        if( LAPACK_DISNAN( x[i] ) )
+    for( i = 0; i < n; i++ ) {
+        if( LAPACK_DISNAN( x[(size_t)i*inc] ) )
             return (lapack_logical) 1;
     }
     return (lapack_logical) 0;