diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 228366345..b7382015d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,4 +1,4 @@ -name: Rocketmq Rust CI +name: RocketMQ Rust CI permissions: contents: read @@ -14,130 +14,178 @@ env: RUST_BACKTRACE: full CI: true + # ===== RocksDB / build cache ===== + CARGO_TARGET_DIR: target + ROCKSDB_DISABLE_JEMALLOC: 1 + jobs: - # Format and lint checks + # ========================= + # Format + Clippy + # ========================= check: name: Check (fmt + clippy) runs-on: ubuntu-latest + steps: - - name: Checkout code + - name: Checkout uses: actions/checkout@v4 - - name: Install dependencies (ubuntu-latest) + - name: Install system dependencies (Ubuntu) run: | - sudo apt-get install protobuf-compiler + sudo apt-get update + sudo apt-get install -y \ + clang llvm libclang-dev \ + cmake make ninja-build pkg-config \ + protobuf-compiler \ + libsnappy-dev liblz4-dev libzstd-dev zlib1g-dev + + echo "CC=clang" >> $GITHUB_ENV + echo "CXX=clang++" >> $GITHUB_ENV + + LIBCLANG_DIR=$(ls -d /usr/lib/llvm-*/lib | head -n 1) + echo "LIBCLANG_PATH=$LIBCLANG_DIR" >> $GITHUB_ENV - name: Set up Rust nightly uses: dtolnay/rust-toolchain@nightly with: components: rustfmt, clippy - - name: Cache Rust dependencies + - name: Cache Rust + RocksDB uses: Swatinem/rust-cache@v2 with: - shared-key: rust-ci - - - name: Install dependencies - run: | - rustup component add rustfmt - rustup component add clippy + shared-key: rocketmq-rust-ubuntu + workspaces: | + . + env-vars: | + CC + CXX + LIBCLANG_PATH + ROCKSDB_DISABLE_JEMALLOC - name: Format check run: cargo fmt --all -- --check - - name: Clippy check (other packages) - run: cargo clippy --all-targets --workspace --exclude rocketmq-controller --all-features -- -D warnings + - name: Clippy (all features) + run: cargo clippy --workspace --all-targets --all-features -- -D warnings - - name: Clippy check (controller without rocksdb) - run: cargo clippy --all-targets -p rocketmq-controller --no-default-features --features storage-file,metrics,debug -- -D warnings - - # Build and test on multiple platforms + # ========================= + # Build + Test + # ========================= build-test: name: Build & Test (${{ matrix.os }}) - runs-on: ${{ matrix.os }} needs: check + runs-on: ${{ matrix.os }} + strategy: - fail-fast: true + fail-fast: false matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] steps: - - name: Checkout code + - name: Checkout uses: actions/checkout@v4 - - if: matrix.os == 'ubuntu-latest' - name: Install dependencies (ubuntu-latest) + # ---------- Ubuntu ---------- + - name: Install system dependencies (Ubuntu) + if: matrix.os == 'ubuntu-latest' run: | - sudo apt-get install protobuf-compiler - - if: matrix.os == 'macos-latest' - name: Install dependencies (macos-latest) + sudo apt-get update + sudo apt-get install -y \ + clang llvm libclang-dev \ + cmake make ninja-build pkg-config \ + protobuf-compiler \ + libsnappy-dev liblz4-dev libzstd-dev zlib1g-dev + + echo "CC=clang" >> $GITHUB_ENV + echo "CXX=clang++" >> $GITHUB_ENV + LIBCLANG_DIR=$(ls -d /usr/lib/llvm-*/lib | head -n 1) + echo "LIBCLANG_PATH=$LIBCLANG_DIR" >> $GITHUB_ENV + + # ---------- macOS ---------- + - name: Install system dependencies (macOS) + if: matrix.os == 'macos-latest' run: | - FILENAME="protoc-3.20.3-osx-x86_64.zip" - URL=https://github.com/protocolbuffers/protobuf/releases/download/v3.20.3/protoc-3.20.3-osx-x86_64.zip - echo "Detected arch: $ARCH, downloading $FILENAME" - curl -LO "$URL" - - sudo unzip -o "$FILENAME" -d /usr/local - protoc --version - - - if: matrix.os == 'windows-latest' - name: Install dependencies (windows-latest) + brew update + brew install cmake protobuf snappy lz4 zstd + + # ---------- Windows ---------- + - name: Install system dependencies (Windows) + if: matrix.os == 'windows-latest' run: | - choco install protoc + choco install cmake ninja protoc -y - name: Set up Rust nightly uses: dtolnay/rust-toolchain@nightly - - name: Cache Rust dependencies + - name: Cache Rust + RocksDB uses: Swatinem/rust-cache@v2 with: - shared-key: rust-ci - - - name: Build (other packages) - run: cargo build --verbose --workspace --exclude rocketmq-controller --all-features - - - name: Build (controller without rocksdb) - run: cargo build --verbose -p rocketmq-controller --no-default-features --features storage-file,metrics,debug - - - name: Run tests (other packages) - run: cargo test --verbose --workspace --exclude rocketmq-controller --all-features - - - name: Run tests (controller without rocksdb) - run: cargo test --verbose -p rocketmq-controller --no-default-features --features storage-file,metrics,debug - - # Code coverage (only on Ubuntu) + shared-key: rocketmq-rust-${{ matrix.os }} + workspaces: | + . + env-vars: | + CC + CXX + LIBCLANG_PATH + ROCKSDB_DISABLE_JEMALLOC + + - name: Build (all features) + run: cargo build --workspace --all-features --verbose + + - name: Test (all features) + run: cargo test --workspace --all-features --verbose + + # ========================= + # Coverage (Ubuntu only) + # ========================= coverage: name: Code Coverage - runs-on: ubuntu-latest needs: check + runs-on: ubuntu-latest + steps: - - name: Checkout code + - name: Checkout uses: actions/checkout@v4 - - name: Install dependencies (ubuntu-latest) - run: sudo apt-get install protobuf-compiler + - name: Install system dependencies (Ubuntu) + run: | + sudo apt-get update + sudo apt-get install -y \ + clang llvm libclang-dev \ + cmake make ninja-build pkg-config \ + protobuf-compiler \ + libsnappy-dev liblz4-dev libzstd-dev zlib1g-dev + + echo "CC=clang" >> $GITHUB_ENV + echo "CXX=clang++" >> $GITHUB_ENV + LIBCLANG_DIR=$(ls -d /usr/lib/llvm-*/lib | head -n 1) + echo "LIBCLANG_PATH=$LIBCLANG_DIR" >> $GITHUB_ENV - name: Set up Rust nightly uses: dtolnay/rust-toolchain@nightly - - name: Cache Rust dependencies + - name: Cache Rust + RocksDB uses: Swatinem/rust-cache@v2 with: - shared-key: rust-ci + shared-key: rocketmq-rust-coverage + workspaces: | + . + env-vars: | + CC + CXX + LIBCLANG_PATH + ROCKSDB_DISABLE_JEMALLOC - name: Install cargo-llvm-cov uses: taiki-e/install-action@cargo-llvm-cov - - name: Generate code coverage (other packages) - run: | - cargo llvm-cov --workspace --exclude rocketmq-controller --all-features --lcov --output-path lcov-other.info - cargo llvm-cov -p rocketmq-controller --no-default-features --features storage-file,metrics,debug --lcov --output-path lcov-controller.info - cat lcov-other.info lcov-controller.info > lcov.info + - name: Generate coverage (all features) + run: cargo llvm-cov --workspace --all-features --lcov --output-path lcov.info - - name: Upload coverage to Codecov + - name: Upload to Codecov uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} files: lcov.info fail_ci_if_error: false - verbose: true \ No newline at end of file + verbose: true diff --git a/Cargo.lock b/Cargo.lock index b29011d68..fd2e85cd0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2910,7 +2910,7 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "raft" version = "0.7.0" -source = "git+https://github.com/tikv/raft-rs.git?branch=master#1fd05e000fb094015507c5c985849d370100fb72" +source = "git+https://github.com/tikv/raft-rs.git?rev=1fd05e0#1fd05e000fb094015507c5c985849d370100fb72" dependencies = [ "bytes", "fxhash", @@ -2928,7 +2928,7 @@ dependencies = [ [[package]] name = "raft-proto" version = "0.7.0" -source = "git+https://github.com/tikv/raft-rs.git?branch=master#1fd05e000fb094015507c5c985849d370100fb72" +source = "git+https://github.com/tikv/raft-rs.git?rev=1fd05e0#1fd05e000fb094015507c5c985849d370100fb72" dependencies = [ "bytes", "protobuf", diff --git a/rocketmq-controller/Cargo.toml b/rocketmq-controller/Cargo.toml index b2b2f4eef..39d49b04c 100644 --- a/rocketmq-controller/Cargo.toml +++ b/rocketmq-controller/Cargo.toml @@ -27,7 +27,7 @@ tokio-util = { workspace = true } # Raft consensus algorithm - MIGRATING TO OPENRAFT # Old raft for legacy code (temporary until migration completes) -raft = { git = "https://github.com/tikv/raft-rs.git", branch = "master" } +raft = { git = "https://github.com/tikv/raft-rs.git", rev = "1fd05e0" } protobuf = "2.28" # New openraft implementation openraft = { git = "https://github.com/databendlabs/openraft.git", branch = "release-0.10", features = ["serde", "tokio-rt"] }