Auto merge of #75621 - TimDiekmann:no-fast-realloc, r=Amanieu

Remove fast path in reallocation for same layout sizes

r? @Amanieu

Before merging a perf-run should be done.

Closes rust-lang/wg-allocators#70

Author: bors

library/alloc/src/alloc.rs

@@ -209,16 +209,14 @@ unsafe impl AllocRef for Global {
         );
 
         // SAFETY: `new_size` must be non-zero, which is checked in the match expression.
+        // If `new_size` is zero, then `old_size` has to be zero as well.
         // Other conditions must be upheld by the caller
         unsafe {
             match layout.size() {
-                old_size if old_size == new_size => {
-                    Ok(NonNull::slice_from_raw_parts(ptr, new_size))
-                }
                 0 => self.alloc(Layout::from_size_align_unchecked(new_size, layout.align())),
                 old_size => {
-                    // `realloc` probably checks for `new_size > size` or something similar.
-                    intrinsics::assume(new_size > old_size);
+                    // `realloc` probably checks for `new_size >= size` or something similar.
+                    intrinsics::assume(new_size >= old_size);
                     let raw_ptr = realloc(ptr.as_ptr(), layout, new_size);
                     let ptr = NonNull::new(raw_ptr).ok_or(AllocErr)?;
                     Ok(NonNull::slice_from_raw_parts(ptr, new_size))
@@ -240,16 +238,14 @@ unsafe impl AllocRef for Global {
         );
 
         // SAFETY: `new_size` must be non-zero, which is checked in the match expression.
+        // If `new_size` is zero, then `old_size` has to be zero as well.
         // Other conditions must be upheld by the caller
         unsafe {
             match layout.size() {
-                old_size if old_size == new_size => {
-                    Ok(NonNull::slice_from_raw_parts(ptr, new_size))
-                }
                 0 => self.alloc_zeroed(Layout::from_size_align_unchecked(new_size, layout.align())),
                 old_size => {
-                    // `realloc` probably checks for `new_size > size` or something similar.
-                    intrinsics::assume(new_size > old_size);
+                    // `realloc` probably checks for `new_size >= size` or something similar.
+                    intrinsics::assume(new_size >= old_size);
                     let raw_ptr = realloc(ptr.as_ptr(), layout, new_size);
                     raw_ptr.add(old_size).write_bytes(0, new_size - old_size);
                     let ptr = NonNull::new(raw_ptr).ok_or(AllocErr)?;
@@ -272,11 +268,8 @@ unsafe impl AllocRef for Global {
             "`new_size` must be smaller than or equal to `layout.size()`"
         );
 
-        let ptr = if new_size == old_size {
-            ptr
-        } else if new_size == 0 {
-            // SAFETY: `layout` is non-zero in size as `old_size` != `new_size`
-            // Other conditions must be upheld by the caller
+        let ptr = if new_size == 0 {
+            // SAFETY: conditions must be upheld by the caller
             unsafe {
                 self.dealloc(ptr, layout);
             }
@@ -285,8 +278,8 @@ unsafe impl AllocRef for Global {
             // SAFETY: new_size is not zero,
             // Other conditions must be upheld by the caller
             let raw_ptr = unsafe {
-                // `realloc` probably checks for `new_size < old_size` or something similar.
-                intrinsics::assume(new_size < old_size);
+                // `realloc` probably checks for `new_size <= old_size` or something similar.
+                intrinsics::assume(new_size <= old_size);
                 realloc(ptr.as_ptr(), layout, new_size)
             };
             NonNull::new(raw_ptr).ok_or(AllocErr)?

library/core/src/alloc/mod.rs

@@ -163,8 +163,6 @@ pub unsafe trait AllocRef {
     /// * `new_size` must be greater than or equal to `layout.size()`, and
     /// * `new_size`, when rounded up to the nearest multiple of `layout.align()`, must not overflow
     ///   (i.e., the rounded value must be less than or equal to `usize::MAX`).
-    // Note: We can't require that `new_size` is strictly greater than `layout.size()` because of ZSTs.
-    // alternative: `new_size` must be strictly greater than `layout.size()` or both are zero
     ///
     /// [*currently allocated*]: #currently-allocated-memory
     /// [*fit*]: #memory-fitting
@@ -194,10 +192,6 @@ pub unsafe trait AllocRef {
             "`new_size` must be greater than or equal to `layout.size()`"
         );
 
-        if size == new_size {
-            return Ok(NonNull::slice_from_raw_parts(ptr, size));
-        }
-
         let new_layout =
             // SAFETY: the caller must ensure that the `new_size` does not overflow.
             // `layout.align()` comes from a `Layout` and is thus guaranteed to be valid for a Layout.
@@ -238,8 +232,6 @@ pub unsafe trait AllocRef {
     /// * `new_size` must be greater than or equal to `layout.size()`, and
     /// * `new_size`, when rounded up to the nearest multiple of `layout.align()`, must not overflow
     ///   (i.e., the rounded value must be less than or equal to `usize::MAX`).
-    // Note: We can't require that `new_size` is strictly greater than `layout.size()` because of ZSTs.
-    // alternative: `new_size` must be strictly greater than `layout.size()` or both are zero
     ///
     /// [*currently allocated*]: #currently-allocated-memory
     /// [*fit*]: #memory-fitting
@@ -269,10 +261,6 @@ pub unsafe trait AllocRef {
             "`new_size` must be greater than or equal to `layout.size()`"
         );
 
-        if size == new_size {
-            return Ok(NonNull::slice_from_raw_parts(ptr, size));
-        }
-
         let new_layout =
             // SAFETY: the caller must ensure that the `new_size` does not overflow.
             // `layout.align()` comes from a `Layout` and is thus guaranteed to be valid for a Layout.
@@ -315,8 +303,6 @@ pub unsafe trait AllocRef {
     /// * `ptr` must denote a block of memory [*currently allocated*] via this allocator,
     /// * `layout` must [*fit*] that block of memory (The `new_size` argument need not fit it.), and
     /// * `new_size` must be smaller than or equal to `layout.size()`.
-    // Note: We can't require that `new_size` is strictly smaller than `layout.size()` because of ZSTs.
-    // alternative: `new_size` must be smaller than `layout.size()` or both are zero
     ///
     /// [*currently allocated*]: #currently-allocated-memory
     /// [*fit*]: #memory-fitting
@@ -346,10 +332,6 @@ pub unsafe trait AllocRef {
             "`new_size` must be smaller than or equal to `layout.size()`"
         );
 
-        if size == new_size {
-            return Ok(NonNull::slice_from_raw_parts(ptr, size));
-        }
-
         let new_layout =
         // SAFETY: the caller must ensure that the `new_size` does not overflow.
         // `layout.align()` comes from a `Layout` and is thus guaranteed to be valid for a Layout.

library/std/src/alloc.rs

@@ -181,16 +181,14 @@ unsafe impl AllocRef for System {
         );
 
         // SAFETY: `new_size` must be non-zero, which is checked in the match expression.
+        // If `new_size` is zero, then `old_size` has to be zero as well.
         // Other conditions must be upheld by the caller
         unsafe {
             match layout.size() {
-                old_size if old_size == new_size => {
-                    Ok(NonNull::slice_from_raw_parts(ptr, new_size))
-                }
                 0 => self.alloc(Layout::from_size_align_unchecked(new_size, layout.align())),
                 old_size => {
-                    // `realloc` probably checks for `new_size > size` or something similar.
-                    intrinsics::assume(new_size > old_size);
+                    // `realloc` probably checks for `new_size >= size` or something similar.
+                    intrinsics::assume(new_size >= old_size);
                     let raw_ptr = GlobalAlloc::realloc(&System, ptr.as_ptr(), layout, new_size);
                     let ptr = NonNull::new(raw_ptr).ok_or(AllocErr)?;
                     Ok(NonNull::slice_from_raw_parts(ptr, new_size))
@@ -212,16 +210,14 @@ unsafe impl AllocRef for System {
         );
 
         // SAFETY: `new_size` must be non-zero, which is checked in the match expression.
+        // If `new_size` is zero, then `old_size` has to be zero as well.
         // Other conditions must be upheld by the caller
         unsafe {
             match layout.size() {
-                old_size if old_size == new_size => {
-                    Ok(NonNull::slice_from_raw_parts(ptr, new_size))
-                }
                 0 => self.alloc_zeroed(Layout::from_size_align_unchecked(new_size, layout.align())),
                 old_size => {
-                    // `realloc` probably checks for `new_size > size` or something similar.
-                    intrinsics::assume(new_size > old_size);
+                    // `realloc` probably checks for `new_size >= size` or something similar.
+                    intrinsics::assume(new_size >= old_size);
                     let raw_ptr = GlobalAlloc::realloc(&System, ptr.as_ptr(), layout, new_size);
                     raw_ptr.add(old_size).write_bytes(0, new_size - old_size);
                     let ptr = NonNull::new(raw_ptr).ok_or(AllocErr)?;
@@ -244,11 +240,8 @@ unsafe impl AllocRef for System {
             "`new_size` must be smaller than or equal to `layout.size()`"
         );
 
-        let ptr = if new_size == old_size {
-            ptr
-        } else if new_size == 0 {
-            // SAFETY: `layout` is non-zero in size as `old_size` != `new_size`
-            // Other conditions must be upheld by the caller
+        let ptr = if new_size == 0 {
+            // SAFETY: conditions must be upheld by the caller
             unsafe {
                 self.dealloc(ptr, layout);
             }
@@ -257,8 +250,8 @@ unsafe impl AllocRef for System {
             // SAFETY: new_size is not zero,
             // Other conditions must be upheld by the caller
             let raw_ptr = unsafe {
-                // `realloc` probably checks for `new_size < old_size` or something similar.
-                intrinsics::assume(new_size < old_size);
+                // `realloc` probably checks for `new_size <= old_size` or something similar.
+                intrinsics::assume(new_size <= old_size);
                 GlobalAlloc::realloc(&System, ptr.as_ptr(), layout, new_size)
             };
             NonNull::new(raw_ptr).ok_or(AllocErr)?