test: fix weak assertions in operators_coverage_tests #106
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [ main, develop ] | |
| pull_request: | |
| branches: [ main, develop ] | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RUST_BACKTRACE: 1 | |
| jobs: | |
| test: | |
| name: Test Suite | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macos-latest] | |
| rust: [stable, beta, nightly] | |
| exclude: | |
| # Reduce CI load by testing nightly only on Ubuntu | |
| - os: windows-latest | |
| rust: nightly | |
| - os: macos-latest | |
| rust: nightly | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| with: | |
| submodules: recursive | |
| - name: Install Protocol Buffers compiler | |
| run: | | |
| if [[ "${{ runner.os }}" == "Linux" ]]; then | |
| sudo apt-get update | |
| sudo apt-get install -y protobuf-compiler | |
| elif [[ "${{ runner.os }}" == "macOS" ]]; then | |
| brew install protobuf | |
| elif [[ "${{ runner.os }}" == "Windows" ]]; then | |
| choco install protoc | |
| fi | |
| shell: bash | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: ${{ matrix.rust == 'stable' && '1.85.0' || matrix.rust }} | |
| components: rustfmt, clippy | |
| - name: Cache cargo registry | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo- | |
| - name: Set feature flags | |
| id: features | |
| shell: bash | |
| run: | | |
| if [[ "${{ runner.os }}" == "Windows" ]]; then | |
| echo "flags=--features async,parallel,naive-conv,formal-verification" >> $GITHUB_OUTPUT | |
| else | |
| echo "flags=--all-features" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Check formatting | |
| run: cargo fmt --all -- --check | |
| if: matrix.rust == 'stable' && matrix.os == 'ubuntu-latest' | |
| - name: Run clippy | |
| run: cargo clippy --all-targets ${{ steps.features.outputs.flags }} -- -D warnings | |
| if: matrix.rust == 'stable' | |
| - name: Build | |
| run: cargo build --verbose ${{ steps.features.outputs.flags }} | |
| - name: Run tests | |
| run: cargo test --verbose ${{ steps.features.outputs.flags }} | |
| - name: Run integration tests | |
| run: cargo test --test integration_tests --verbose | |
| - name: Test examples | |
| run: | | |
| cargo run --example simple_model | |
| cargo run --example tensor_ops | |
| - name: Test CLI runner | |
| run: | | |
| # Test with sample data if available | |
| if [ -f "test_input.json" ]; then | |
| echo "Testing CLI runner with sample input..." | |
| # This would need a valid ONNX model file to work properly | |
| # cargo run --bin runnx-runner -- --help | |
| fi | |
| shell: bash | |
| benchmark: | |
| name: Performance Benchmarks | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| with: | |
| submodules: recursive | |
| - name: Install Protocol Buffers compiler | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y protobuf-compiler | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: stable | |
| - name: Cache cargo registry | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ubuntu-latest-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Run benchmarks | |
| run: cargo bench | |
| - name: Upload benchmark results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results | |
| path: target/criterion/ | |
| security: | |
| name: Security Audit | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| with: | |
| submodules: recursive | |
| - name: Install Protocol Buffers compiler | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y protobuf-compiler | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: stable | |
| - name: Cache cargo registry | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ubuntu-latest-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Install cargo-audit | |
| run: cargo install cargo-audit | |
| - name: Run security audit | |
| run: cargo audit | |
| docs: | |
| name: Documentation | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| with: | |
| submodules: recursive | |
| - name: Install Protocol Buffers compiler | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y protobuf-compiler | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: stable | |
| - name: Cache cargo registry | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ubuntu-latest-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Build documentation | |
| run: cargo doc --all-features --no-deps --document-private-items | |
| - name: Upload documentation | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: documentation | |
| path: target/doc/ | |
| msrv: | |
| name: Minimum Supported Rust Version | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| with: | |
| submodules: recursive | |
| - name: Install Protocol Buffers compiler | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y protobuf-compiler | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: 1.85 # Full support for edition 2024 | |
| - name: Cache cargo registry | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: msrv-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Test with MSRV | |
| run: cargo test --all-features | |
| coverage: | |
| name: Code Coverage Analysis | |
| runs-on: ubuntu-latest | |
| # Only run coverage on stable Rust and Ubuntu for efficiency | |
| if: github.event_name == 'push' || github.event_name == 'pull_request' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| with: | |
| submodules: recursive | |
| - name: Install Protocol Buffers compiler | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y protobuf-compiler | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: stable | |
| components: llvm-tools-preview | |
| - name: Cache cargo registry | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: coverage-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| coverage-${{ runner.os }}-cargo- | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: cargo-llvm-cov | |
| - name: Make coverage script executable | |
| run: chmod +x scripts/coverage.sh | |
| - name: Display coverage configuration | |
| run: | | |
| echo "📊 Coverage Configuration" | |
| echo "=========================" | |
| echo "Exclusion file: .llvm-cov-exclude" | |
| echo "" | |
| echo "Excluded patterns:" | |
| cat .llvm-cov-exclude | grep -v '^#' | grep -v '^$' | sed 's/^/ - /' | |
| echo "" | |
| - name: Build runner binary for CLI tests | |
| run: | | |
| echo "🔨 Building runner binary for CLI tests..." | |
| cargo build --bin runnx-runner | |
| - name: Run comprehensive coverage analysis | |
| run: | | |
| echo "🧪 Running all tests with coverage..." | |
| ./scripts/coverage.sh --all-features --workspace --no-cfg-coverage | |
| - name: Generate coverage reports (multiple formats) | |
| run: | | |
| echo "📈 Generating coverage reports..." | |
| ./scripts/coverage.sh --all-features --workspace --lcov --output-path lcov.info --no-cfg-coverage | |
| ./scripts/coverage.sh --all-features --workspace --json --output-path coverage.json --no-cfg-coverage | |
| ./scripts/coverage.sh --all-features --workspace --html --output-dir target/coverage-html --no-cfg-coverage | |
| - name: Extract coverage percentage | |
| id: coverage | |
| run: | | |
| COVERAGE=$(./scripts/coverage.sh --all-features --workspace --summary-only --no-cfg-coverage 2>/dev/null | grep "TOTAL" | awk '{print $4}' | head -1) | |
| echo "percentage=$COVERAGE" >> $GITHUB_OUTPUT | |
| echo "Coverage: $COVERAGE" | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v5 | |
| with: | |
| files: lcov.info | |
| fail_ci_if_error: false | |
| verbose: true | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Upload coverage reports (artifacts) | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-reports-${{ github.run_number }} | |
| path: | | |
| lcov.info | |
| coverage.json | |
| target/coverage-html/ | |
| retention-days: 30 | |
| - name: Coverage summary comment (PR only) | |
| if: github.event_name == 'pull_request' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const coverage = '${{ steps.coverage.outputs.percentage }}'; | |
| const comment = `## 📊 Code Coverage Report | |
| **Overall Coverage: ${coverage}** | |
| 📄 **Coverage Reports Available:** | |
| - [Download HTML Report](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) | |
| - [View on Codecov](https://codecov.io/gh/${{ github.repository }}/pull/${{ github.event.number }}) | |
| 🔍 **Exclusions Applied:** | |
| - \`src/onnx.rs\` (auto-generated protobuf bindings) | |
| - \`src/bin/runner.rs\` (CLI entry point) | |
| - \`build.rs\` (build script) | |
| - \`src/formal.rs\` (formal specification module) | |
| This report was generated using our custom coverage script with configuration-based exclusions.`; | |
| github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: comment | |
| }); | |
| - name: Check coverage threshold | |
| run: | | |
| COVERAGE=$(echo "${{ steps.coverage.outputs.percentage }}" | sed 's/%//') | |
| THRESHOLD=75 | |
| if (( $(echo "$COVERAGE >= $THRESHOLD" | bc -l) )); then | |
| echo "✅ Coverage $COVERAGE% meets threshold of $THRESHOLD%" | |
| else | |
| echo "❌ Coverage $COVERAGE% below threshold of $THRESHOLD%" | |
| echo "Please add tests to improve coverage" | |
| exit 1 | |
| fi |