rust: Define cabi_realloc as a weak symbol (bytecodealliance#851)

* rust: Define `cabi_realloc` as a weak symbol

This commit updates the `wit-bindgen` Rust crate to define the
`cabi_realloc` symbol as a weak symbol. This is not easy because Rust
does not offer a stable means by which to do this. Despite this through
a combination of shell scripts and CI it should be possible to get this
all working together by using C to define a weak symbol calling a known
Rust symbol.

Closes bytecodealliance#849

* Fix typo in yml

* More yml changes

* Regenerate cabi_realloc files

* Strip custom sections

* Install wasm-tools on CI as well

* Pin to slightly older wasm-tools

* Fix native tests

* Subvert rate limiting

.github/actions/install-wasi-sdk/action.yml

@@ -0,0 +1,26 @@
+name: 'Install the wasi-sdk'
+description: 'Install the wasi-sdk toolchain'
+
+runs:
+  using: composite
+  steps:
+    - run: |
+        curl https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-linux.tar.gz -L | tar xzvf -
+        echo "WASI_SDK_PATH=`pwd`/wasi-sdk-20.0" >> $GITHUB_ENV
+      if: runner.os == 'Linux'
+      shell: bash
+    - run: |
+        curl https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-macos.tar.gz -L | tar xzvf -
+        echo "WASI_SDK_PATH=`pwd`/wasi-sdk-20.0" >> $GITHUB_ENV
+      if: runner.os == 'macOS'
+      shell: bash
+    - run: |
+        curl https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0.m-mingw.tar.gz -L | tar xzvf -
+        echo "WASI_SDK_PATH=`pwd`/wasi-sdk-20.0+m" >> $GITHUB_ENV
+      if: runner.os == 'Windows'
+      shell: bash
+    - name: Setup `wasm-tools`
+      uses: bytecodealliance/actions/wasm-tools/setup@v1
+      with:
+        version: "1.0.60"
+        github_token: ${{ github.token }}

.github/workflows/main.yml

@@ -67,18 +67,7 @@ jobs:
     - name: Install wasm32-wasi target
       run: rustup target add wasm32-wasi
 
-    - run: |
-        curl https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-linux.tar.gz -L | tar xzvf -
-        echo "WASI_SDK_PATH=`pwd`/wasi-sdk-20.0" >> $GITHUB_ENV
-      if : matrix.os == 'ubuntu-latest'
-    - run: |
-        curl https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-macos.tar.gz -L | tar xzvf -
-        echo "WASI_SDK_PATH=`pwd`/wasi-sdk-20.0" >> $GITHUB_ENV
-      if : matrix.os == 'macos-latest'
-    - run: |
-        curl https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0.m-mingw.tar.gz -L | tar xzvf -
-        echo "WASI_SDK_PATH=`pwd`/wasi-sdk-20.0+m" >> $GITHUB_ENV
-      if : matrix.os == 'windows-latest'
+    - uses: ./.github/actions/install-wasi-sdk
 
     - run: |
         curl.exe -LO https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.ps1
@@ -119,6 +108,26 @@ jobs:
       run: |
         source ./emsdk-main/emsdk_env.sh
         cargo test --workspace
+
+  check:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install Rust
+      run: rustup update stable --no-self-update && rustup default stable
+    - name: Install wasm32-unknown-unknown target
+      run: rustup target add wasm32-unknown-unknown
+    - name: Install wasm32-wasi target
+      run: rustup target add wasm32-wasi
+
+    # Verify the output of the `./ci/rebuild-libcabi-realloc.sh` script is
+    # up-to-date.
+    - uses: ./.github/actions/install-wasi-sdk
+    - run: ./ci/rebuild-libcabi-realloc.sh
+    - run: git diff --exit-code
+
+    # Test various feature combinations, make sure they all build
     - run: cargo build
     - run: cargo build --no-default-features
     - run: cargo build --no-default-features --features rust
@@ -128,7 +137,6 @@ jobs:
     - run: cargo build --no-default-features --features csharp
     - run: cargo build --no-default-features --features markdown
 
-
   rustfmt:
     name: Rustfmt
     runs-on: ubuntu-latest
@@ -162,6 +170,7 @@ jobs:
       - rustfmt
       - build
       - verify-publish
+      - check
     if: always()
 
     steps:

.github/workflows/release-process.yml

@@ -34,6 +34,11 @@ jobs:
       - name: Bump version number
         run: ./publish ${{ github.event.inputs.action }}
 
+      # Regenerate precompiled artifacts in this repo as they're dependent on
+      # the version number.
+      - uses: ./.github/actions/install-wasi-sdk
+      - run: ./ci/rebuild-libcabi-realloc.sh
+
       - name: Prep PR metadata
         run: |
           set -ex

.gitignore

@@ -7,3 +7,4 @@ ace
 *.wasm
 !wasi_snapshot_preview1.reactor.wasm
 __pycache__
+crates/guest-rust/src/cabi_realloc.o

ci/rebuild-libcabi-realloc.sh

@@ -0,0 +1,85 @@
+#!/bin/sh
+
+# This script, and various infrastructure, is a hack to work around the lack of
+# stable support in Rust to generate a weak symbol.
+#
+# The basic problem here is that the Rust `wit-bindgen` crate wants to export
+# the `cabi_realloc` symbol from the final binary. This library, however,
+# is not stable which means that we're publishing new versions of `wit-bindgen`
+# over its development. This means that if `wit-bindgen`-the-crate were to
+# export a `#[no_mangle]` symbol of `cabi_realloc` then it wouldn't work to
+# have two versions of `wit-bindgen` in the same project. This can arise
+# relatively quickly, however, so this is something we want to solve.
+#
+# The general idea of the solution here is to ensure that the `cabi_realloc`
+# symbol itself is declared as a weak symbol. A weakly-defined symbol means
+# that if the linker sees multiple copies it can discard all but one. This is
+# the semantics we want where some `wit-bindgen` needs to define `cabi_realloc`
+# but it doesn't matter much which one.
+#
+# Stable Rust can't define weak symbols as of the time of this writing. C,
+# however, can. Unfortunately users of this crate do not always have a C
+# compiler on-hand for wasm, nor do we want to require one. That's where all
+# these hacks come into play. With that intro, the purpose of this script is to:
+#
+# * Generate a `cabi_realloc.rs` file with a "mangled" Rust symbol that's
+#   unique per-major-version of the crate.
+# * Generate a `cabi_realloc.c` file that defines a weak `cabi_realloc` symbol
+#   that calls the above Rust symbol
+# * Compile `cabi_realloc.c` into an object and place it into an archive and
+#   check that archive into this repo.
+#
+# This all leads up to the point where we're distributing binary artifacts with
+# this crate. These artifacts are verified in CI to ensure what this script
+# generates.
+#
+# Overall this is intended to provide `cabi_realloc` as a weak symbol,
+# everything works on stable Rust, and users don't need a C compiler when they
+# use this crate.
+
+set -ex
+
+version=$(grep '^version =' ./Cargo.toml | sed 's/.*"\(.*\)"/\1/' | sed 's/\./_/g')
+
+sym=cabi_realloc_wit_bindgen_$version
+
+cat >./crates/guest-rust/src/cabi_realloc.rs <<-EOF
+// This file is generated by $0
+
+#[no_mangle]
+pub unsafe extern "C" fn $sym(
+    old_ptr: *mut u8,
+    old_len: usize,
+    align: usize,
+    new_len: usize,
+) -> *mut u8 {
+    crate::rt::cabi_realloc(old_ptr, old_len, align, new_len)
+}
+EOF
+
+cat >./crates/guest-rust/src/cabi_realloc.c <<-EOF
+// This file is generated by $0
+
+#include <stdint.h>
+
+extern void *$sym(void *ptr, size_t old_size, size_t align, size_t new_size);
+
+__attribute__((__weak__, __export_name__("cabi_realloc")))
+void *cabi_realloc(void *ptr, size_t old_size, size_t align, size_t new_size) {
+  return $sym(ptr, old_size, align, new_size);
+}
+EOF
+
+rm -f crates/guest-rust/src/cabi_realloc.o
+$WASI_SDK_PATH/bin/clang crates/guest-rust/src/cabi_realloc.c \
+  -O -c -o crates/guest-rust/src/cabi_realloc.o
+
+# Remove the `producers` section. This appears to differ whether the host for
+# clang is either macOS or Linux. Not needed here anyway, so discard it to help
+# either host produce the same object.
+wasm-tools strip -d producers ./crates/guest-rust/src/cabi_realloc.o \
+  -o ./crates/guest-rust/src/cabi_realloc.o
+
+rm -f crates/guest-rust/src/libwit_bindgen_cabi_realloc.a
+$WASI_SDK_PATH/bin/llvm-ar crus crates/guest-rust/src/libwit_bindgen_cabi_realloc.a \
+  crates/guest-rust/src/cabi_realloc.o

crates/guest-rust/build.rs

@@ -0,0 +1,21 @@
+fn main() {
+    if !cfg!(feature = "realloc") {
+        return;
+    }
+
+    let target_arch = std::env::var("CARGO_CFG_TARGET_ARCH").unwrap_or(String::new());
+    let target_family = std::env::var("CARGO_CFG_TARGET_FAMILY").unwrap_or(String::new());
+
+    if target_family != "wasm" {
+        return;
+    }
+
+    if target_arch != "wasm32" {
+        panic!("only wasm32 supports cabi-realloc right now");
+    }
+
+    println!("cargo:rustc-link-lib=wit_bindgen_cabi_realloc");
+    let cwd = std::env::current_dir().unwrap();
+    let cwd = cwd.display();
+    println!("cargo:rustc-link-search=native={cwd}/src");
+}

crates/guest-rust/src/cabi_realloc.c

@@ -0,0 +1,10 @@
+// This file is generated by ./ci/rebuild-libcabi-realloc.sh
+
+#include <stdint.h>
+
+extern void *cabi_realloc_wit_bindgen_0_18_0(void *ptr, size_t old_size, size_t align, size_t new_size);
+
+__attribute__((__weak__, __export_name__("cabi_realloc")))
+void *cabi_realloc(void *ptr, size_t old_size, size_t align, size_t new_size) {
+  return cabi_realloc_wit_bindgen_0_18_0(ptr, old_size, align, new_size);
+}

crates/guest-rust/src/cabi_realloc.rs

@@ -0,0 +1,11 @@
+// This file is generated by ./ci/rebuild-libcabi-realloc.sh
+
+#[no_mangle]
+pub unsafe extern "C" fn cabi_realloc_wit_bindgen_0_18_0(
+    old_ptr: *mut u8,
+    old_len: usize,
+    align: usize,
+    new_len: usize,
+) -> *mut u8 {
+    crate::rt::cabi_realloc(old_ptr, old_len, align, new_len)
+}

crates/guest-rust/src/lib.rs

@@ -305,6 +305,10 @@ pub use wit_bindgen_rust_macro::generate;
 #[doc(hidden)]
 pub use bitflags;
 
+/// For more information about this see `./ci/rebuild-libcabi-realloc.sh`.
+#[cfg(feature = "realloc")]
+mod cabi_realloc;
+
 #[doc(hidden)]
 pub mod rt {
     use crate::alloc::string::String;
@@ -346,9 +350,36 @@ pub mod rt {
     // Re-export things from liballoc for convenient use.
     pub use super::alloc::{alloc, boxed, string, vec};
 
+    /// This function is called from generated bindings and will be deleted by
+    /// the linker. The purpose of this function is to force a reference to the
+    /// symbol `cabi_realloc` to make its way through to the final linker
+    /// command line. That way `wasm-ld` will pick it up, see it needs to be
+    /// exported, and then export it.
+    ///
+    /// For more information about this see `./ci/rebuild-libcabi-realloc.sh`.
+    pub fn maybe_link_cabi_realloc() {
+        #[cfg(target_family = "wasm")]
+        {
+            #[cfg(feature = "realloc")]
+            extern "C" {
+                fn cabi_realloc(
+                    old_ptr: *mut u8,
+                    old_len: usize,
+                    align: usize,
+                    new_len: usize,
+                ) -> *mut u8;
+            }
+            #[cfg(feature = "realloc")]
+            static _X: unsafe extern "C" fn(*mut u8, usize, usize, usize) -> *mut u8 = cabi_realloc;
+        }
+    }
+
+    /// NB: this function is called by a generated function in the
+    /// `cabi_realloc` module above. It's otherwise never explicitly called.
+    ///
+    /// For more information about this see `./ci/rebuild-libcabi-realloc.sh`.
     #[cfg(feature = "realloc")]
-    #[no_mangle]
-    unsafe extern "C" fn cabi_realloc(
+    pub unsafe fn cabi_realloc(
         old_ptr: *mut u8,
         old_len: usize,
         align: usize,

crates/rust/src/lib.rs

@@ -558,13 +558,17 @@ impl WorldGenerator for RustWasm {
         ));
         self.src.push_str(&format!("{:?};\n", component_type));
 
-        self.src.push_str(
+        let rt = self.runtime_path().to_string();
+        uwriteln!(
+            self.src,
             "
-            #[inline(never)]
-            #[doc(hidden)]
-            #[cfg(target_arch = \"wasm32\")]
-            pub fn __link_section() {}
-        ",
+                #[inline(never)]
+                #[doc(hidden)]
+                #[cfg(target_arch = \"wasm32\")]
+                pub fn __link_section() {{
+                    {rt}::maybe_link_cabi_realloc();
+                }}
+            ",
         );
 
         if self.opts.stubs {