Merge pull request #5835 from changangela/fix-lapacke-nancheck-overflow

Fix 32-bit integer overflow in LAPACKE nancheck index arithmetic

Author: martin-frbg

lapack-netlib/LAPACKE/utils/lapacke_c_nancheck.c

@@ -42,8 +42,8 @@ lapack_logical LAPACKE_c_nancheck( lapack_int n,
     if( incx == 0 ) return (lapack_logical) LAPACK_CISNAN( x[0] );
     inc = ( incx > 0 ) ? incx : -incx ;
 
-    for( i = 0; i < n*inc; i+=inc ) {
-        if( LAPACK_CISNAN( x[i] ) )
+    for( i = 0; i < n; i++ ) {
+        if( LAPACK_CISNAN( x[(size_t)i*inc] ) )
             return (lapack_logical) 1;
     }
     return (lapack_logical) 0;

lapack-netlib/LAPACKE/utils/lapacke_chp_nancheck.c

@@ -39,6 +39,6 @@
 lapack_logical LAPACKE_chp_nancheck( lapack_int n,
                                       const lapack_complex_float *ap )
 {
-    lapack_int len = n*(n+1)/2;
+    lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
     return LAPACKE_c_nancheck( len, ap, 1 );
 }

lapack-netlib/LAPACKE/utils/lapacke_cpf_nancheck.c

@@ -40,6 +40,6 @@
 lapack_logical LAPACKE_cpf_nancheck( lapack_int n,
                                       const lapack_complex_float *a )
 {
-    lapack_int len = n*(n+1)/2;
+    lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
     return LAPACKE_c_nancheck( len, a, 1 );
 }

lapack-netlib/LAPACKE/utils/lapacke_cpp_nancheck.c

@@ -39,6 +39,6 @@
 lapack_logical LAPACKE_cpp_nancheck( lapack_int n,
                                       const lapack_complex_float *ap )
 {
-    lapack_int len = n*(n+1)/2;
+    lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
     return LAPACKE_c_nancheck( len, ap, 1 );
 }

lapack-netlib/LAPACKE/utils/lapacke_csp_nancheck.c

@@ -39,6 +39,6 @@
 lapack_logical LAPACKE_csp_nancheck( lapack_int n,
                                       const lapack_complex_float *ap )
 {
-    lapack_int len = n*(n+1)/2;
+    lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
     return LAPACKE_c_nancheck( len, ap, 1 );
 }

lapack-netlib/LAPACKE/utils/lapacke_ctf_nancheck.c

@@ -150,7 +150,7 @@ lapack_logical LAPACKE_ctf_nancheck( int matrix_layout, char transr,
         }
     } else {
         /* Non-unit case - just check whole array for NaNs. */
-        len = n*(n+1)/2;
+        len = (lapack_int)(((size_t)n*(n+1))/2);
         return LAPACKE_cge_nancheck( LAPACK_COL_MAJOR, len, 1, a, len );
     }
 }

lapack-netlib/LAPACKE/utils/lapacke_ctp_nancheck.c

@@ -76,7 +76,7 @@ lapack_logical LAPACKE_ctp_nancheck( int matrix_layout, char uplo, char diag,
         return (lapack_logical) 0;
     } else {
         /* Non-unit case - just check whole array for NaNs. */
-        len = n*(n+1)/2;
+        len = (lapack_int)(((size_t)n*(n+1))/2);
         return LAPACKE_c_nancheck( len, ap, 1 );
     }
 }

lapack-netlib/LAPACKE/utils/lapacke_ctr_nancheck.c

@@ -68,14 +68,14 @@ lapack_logical LAPACKE_ctr_nancheck( int matrix_layout, char uplo, char diag,
     if( ( colmaj || lower ) && !( colmaj && lower ) ) {
         for( j = st; j < n; j++ ) {
             for( i = 0; i < MIN( j+1-st, lda ); i++ ) {
-                if( LAPACK_CISNAN( a[i+j*lda] ) )
+                if( LAPACK_CISNAN( a[i+(size_t)j*lda] ) )
                     return (lapack_logical) 1;
             }
         }
     } else {
         for( j = 0; j < n-st; j++ ) {
             for( i = j+st; i < MIN( n, lda ); i++ ) {
-                if( LAPACK_CISNAN( a[i+j*lda] ) )
+                if( LAPACK_CISNAN( a[i+(size_t)j*lda] ) )
                     return (lapack_logical) 1;
             }
         }

lapack-netlib/LAPACKE/utils/lapacke_ctz_nancheck.c

@@ -103,27 +103,27 @@ lapack_logical LAPACKE_ctz_nancheck( int matrix_layout, char direct, char uplo,
     }
 
     /* Initial offsets and sizes of triangular and rectangular parts */
-    lapack_int tri_offset = 0;
+    int64_t tri_offset = 0;
     lapack_int tri_n = MIN(m,n);
-    lapack_int rect_offset = -1;
+    int64_t rect_offset = -1;
     lapack_int rect_m = ( m > n ) ? m - n : m;
     lapack_int rect_n = ( n > m ) ? n - m : n;
 
     /* Fix offsets depending on the shape of the matrix */
     if( front ) {
         if( lower && m > n ) {
-            rect_offset = tri_n * ( !colmaj ? lda : 1 );
+            rect_offset = (int64_t)tri_n * ( !colmaj ? lda : 1 );
         } else if( !lower && n > m ) {
-            rect_offset = tri_n * ( colmaj ? lda : 1 );
+            rect_offset = (int64_t)tri_n * ( colmaj ? lda : 1 );
         }
     } else {
         if( m > n ) {
-            tri_offset = rect_m * ( !colmaj ? lda : 1 );
+            tri_offset = (int64_t)rect_m * ( !colmaj ? lda : 1 );
             if( !lower ) {
                 rect_offset = 0;
             }
         } else if( n > m ) {
-            tri_offset = rect_n * ( colmaj ? lda : 1 );
+            tri_offset = (int64_t)rect_n * ( colmaj ? lda : 1 );
             if( lower ) {
                 rect_offset = 0;
             }

lapack-netlib/LAPACKE/utils/lapacke_d_nancheck.c

@@ -42,8 +42,8 @@ lapack_logical LAPACKE_d_nancheck( lapack_int n,
     if( incx == 0 ) return (lapack_logical) LAPACK_DISNAN( x[0] );
     inc = ( incx > 0 ) ? incx : -incx ;
 
-    for( i = 0; i < n*inc; i+=inc ) {
-        if( LAPACK_DISNAN( x[i] ) )
+    for( i = 0; i < n; i++ ) {
+        if( LAPACK_DISNAN( x[(size_t)i*inc] ) )
             return (lapack_logical) 1;
     }
     return (lapack_logical) 0;