Nightly #38
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 6 * * *" | |
| concurrency: | |
| # Use github.run_id on main branch | |
| # Use github.event.pull_request.number on pull requests, so it's unique per pull request | |
| # Use github.ref on other branches, so it's unique per branch | |
| group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| build: | |
| name: Build crud-bench | |
| runs-on: [runner-amd64-4xlarge] | |
| steps: | |
| - name: Install stable toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Checkout sources | |
| uses: actions/checkout@v4 | |
| - name: Setup cache | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| cache-on-failure: true | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - name: Build benchmark | |
| run: cargo build --release --target x86_64-unknown-linux-gnu | |
| - name: Store artifacts | |
| run: cp target/x86_64-unknown-linux-gnu/release/crud-bench crud-bench | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: crud-bench | |
| path: crud-bench | |
| benchmark: | |
| name: Benchmark ${{ matrix.description }} | |
| needs: build | |
| runs-on: [runner-benchmarking] | |
| continue-on-error: true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # ArangoDB | |
| - name: arangodb | |
| database: arangodb | |
| enabled: true | |
| description: ArangoDB | |
| # Cassandra | |
| - name: cassandra | |
| database: cassandra | |
| enabled: false | |
| description: Cassandra | |
| skipped: Cassandra benchmark not yet implemented | |
| # Dragonfly | |
| - name: dragonfly | |
| database: dragonfly | |
| enabled: true | |
| description: Dragonfly | |
| # Dry | |
| - name: dry | |
| database: dry | |
| enabled: true | |
| description: Dry | |
| # Fjall | |
| - name: fjall | |
| database: fjall | |
| enabled: true | |
| description: Fjall | |
| # KeyDB | |
| - name: keydb | |
| database: keydb | |
| enabled: true | |
| description: KeyDB | |
| # LMDB | |
| - name: lmdb | |
| database: lmdb | |
| enabled: true | |
| description: LMDB | |
| # Map | |
| - name: map | |
| database: map | |
| enabled: true | |
| description: Map | |
| # MongoDB | |
| - name: mongodb | |
| database: mongodb | |
| enabled: true | |
| description: MongoDB | |
| # MySQL | |
| - name: mysql | |
| database: mysql | |
| enabled: true | |
| description: MySQL | |
| # Neo4j | |
| - name: neo4j | |
| database: neo4j | |
| enabled: true | |
| description: Neo4j | |
| # Postgres | |
| - name: postgres | |
| database: postgres | |
| enabled: true | |
| description: Postgres | |
| # Redb | |
| - name: redb | |
| database: redb | |
| enabled: false | |
| description: ReDB | |
| skipped: ReDB benchmark skipped due to excessive benchmark time | |
| # Redis | |
| - name: redis | |
| database: redis | |
| enabled: true | |
| description: Redis | |
| # RocksDB | |
| - name: rocksdb | |
| database: rocksdb | |
| enabled: true | |
| description: RocksDB | |
| # Scylladb | |
| - name: scylladb | |
| database: scylladb | |
| enabled: false | |
| description: ScyllaDB | |
| skipped: ScyllaDB benchmark not yet implemented | |
| # SQLite | |
| - name: sqlite | |
| database: sqlite | |
| enabled: true | |
| description: SQLite | |
| # SurrealDB + Memory | |
| - name: surrealdb-memory | |
| database: surrealdb-memory | |
| enabled: true | |
| description: SurrealDB with in-memory storage | |
| # SurrealDB + RocksDB | |
| - name: surrealdb-rocksdb | |
| database: surrealdb-rocksdb | |
| enabled: true | |
| description: SurrealDB with RocksDB storage | |
| # SurrealDB + SurrealKV | |
| - name: surrealdb-surrealkv | |
| database: surrealdb-surrealkv | |
| enabled: true | |
| description: SurrealDB with SurrealKV storage | |
| # SurrealDB Memory Engine | |
| - name: surrealdb-embedded-memory | |
| database: surrealdb | |
| enabled: true | |
| endpoint: -e memory | |
| description: SurrealDB embedded with in-memory storage | |
| # SurrealDB RocksDB Engine | |
| - name: surrealdb-embedded-rocksdb | |
| database: surrealdb | |
| enabled: true | |
| endpoint: -e rocksdb:/tmp/rocksdb-engine | |
| description: SurrealDB embedded with RocksDB storage | |
| # SurrealDB SurrealKV Engine | |
| - name: surrealdb-embedded-surrealkv | |
| database: surrealdb | |
| enabled: true | |
| endpoint: -e surrealkv:/tmp/surrealkv-engine | |
| description: SurrealDB embedded with SurrealKV storage | |
| # SurrealKV | |
| - name: surrealkv | |
| database: surrealkv | |
| enabled: true | |
| description: SurrealKV | |
| steps: | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: ${{ github.workspace }}/artifacts | |
| merge-multiple: true | |
| - name: Set file permissions | |
| run: chmod +x ${{ github.workspace }}/artifacts/crud-bench | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USER }} | |
| password: ${{ secrets.DOCKER_TOKEN }} | |
| - name: System Information | |
| run: | | |
| echo "=== Environment Variables ===" | |
| env | |
| echo "=== Kernel & OS Info ===" | |
| uname -a | |
| echo "=== CPU Details (lscpu) ===" | |
| lscpu | |
| echo "=== First 50 lines of /proc/cpuinfo ===" | |
| head -n 50 /proc/cpuinfo | |
| echo "=== First 50 lines of /proc/meminfo ===" | |
| head -n 50 /proc/meminfo | |
| echo "=== Cgroup Information (/proc/self/cgroup) ===" | |
| cat /proc/self/cgroup | |
| - name: ${{ matrix.skipped || 'Benchmark processing' }} | |
| if: ${{ !matrix.enabled }} | |
| run: echo "${{ matrix.skipped }}" | |
| - name: Clean up environment | |
| if: ${{ matrix.enabled }} | |
| run: | | |
| # Clean up data directory | |
| rm -rf /tmp/crud-bench | |
| mkdir /tmp/crud-bench | |
| chmod 777 /tmp/crud-bench | |
| # Remove old results | |
| rm -f result*.json | |
| rm -f result*.csv | |
| - name: Optimise system | |
| if: ${{ matrix.enabled }} | |
| run: | | |
| # Flush disk writes | |
| sync | |
| # Increase max limits | |
| ulimit -n 65536 | |
| ulimit -u unlimited | |
| ulimit -l unlimited | |
| - name: Run benchmarks (1,000,000 samples / 128 clients / 48 threads / key integer / random) | |
| timeout-minutes: 60 | |
| if: ${{ matrix.enabled && (success() || failure()) }} | |
| run: | | |
| ${{ github.workspace }}/artifacts/crud-bench -d ${{ matrix.database }} ${{ matrix.endpoint || '' }} -s 1000000 -c 128 -t 48 -k integer -r -n integer-random | |
| docker container kill crud-bench &>/dev/null || docker container prune --force &>/dev/null || true | |
| env: | |
| CRUD_BENCH_LMDB_DATABASE_SIZE: 1073741824 # 1 GiB | |
| - name: Run benchmarks (1,000,000 samples / 128 clients / 48 threads / key string26 / random) | |
| timeout-minutes: 60 | |
| if: ${{ matrix.enabled && (success() || failure()) }} | |
| run: | | |
| ${{ github.workspace }}/artifacts/crud-bench -d ${{ matrix.database }} ${{ matrix.endpoint || '' }} -s 1000000 -c 128 -t 48 -k string26 -r -n string26-random | |
| docker container kill crud-bench &>/dev/null || docker container prune --force &>/dev/null || true | |
| env: | |
| CRUD_BENCH_LMDB_DATABASE_SIZE: 1073741824 # 1 GiB | |
| - name: Run benchmarks (1,000,000 samples / 128 clients / 48 threads / key string90 / random) | |
| timeout-minutes: 60 | |
| if: ${{ matrix.enabled && (success() || failure()) }} | |
| run: | | |
| ${{ github.workspace }}/artifacts/crud-bench -d ${{ matrix.database }} ${{ matrix.endpoint || '' }} -s 1000000 -c 128 -t 48 -k string90 -r -n string90-random | |
| docker container kill crud-bench &>/dev/null || docker container prune --force &>/dev/null || true | |
| env: | |
| CRUD_BENCH_LMDB_DATABASE_SIZE: 3221225472 # 3 GiB | |
| - name: Run benchmarks (1,000,000 samples / 128 clients / 48 threads / key string250 / random) | |
| timeout-minutes: 60 | |
| if: ${{ matrix.enabled && (success() || failure()) }} | |
| run: | | |
| ${{ github.workspace }}/artifacts/crud-bench -d ${{ matrix.database }} ${{ matrix.endpoint || '' }} -s 1000000 -c 128 -t 48 -k string250 -r -n string250-random | |
| docker container kill crud-bench &>/dev/null || docker container prune --force &>/dev/null || true | |
| env: | |
| CRUD_BENCH_LMDB_DATABASE_SIZE: 10737418240 # 10 GiB | |
| - name: Run benchmarks (1,000,000 samples / 128 clients / 48 threads / key string506 / random) | |
| timeout-minutes: 60 | |
| if: ${{ matrix.enabled && (success() || failure()) }} | |
| run: | | |
| ${{ github.workspace }}/artifacts/crud-bench -d ${{ matrix.database }} ${{ matrix.endpoint || '' }} -s 1000000 -c 128 -t 48 -k string506 -r -n string506-random | |
| docker container kill crud-bench &>/dev/null || docker container prune --force &>/dev/null || true | |
| env: | |
| CRUD_BENCH_LMDB_DATABASE_SIZE: 16106127360 # 15 GiB | |
| - name: Run benchmarks (1,000,000 samples / 128 clients / 48 threads / key string26 / random / 1.5KiB row and object size) | |
| timeout-minutes: 60 | |
| if: ${{ matrix.enabled && (success() || failure()) }} | |
| run: | | |
| ${{ github.workspace }}/artifacts/crud-bench -d ${{ matrix.database }} ${{ matrix.endpoint || '' }} -s 1000000 -c 128 -t 48 -k string26 -r -n string26-random-1k | |
| docker container kill crud-bench &>/dev/null || docker container prune --force &>/dev/null || true | |
| env: | |
| CRUD_BENCH_LMDB_DATABASE_SIZE: 32212254720 # 30 GiB | |
| CRUD_BENCH_VALUE: '{ "text": "text:50", "integer": "int", "nested": { "text": "text:1000", "array": [ "string:50", "string:50", "string:50", "string:50", "string:50" ] } }' | |
| - name: Upload result artifacts | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ matrix.enabled && (success() || failure()) }} | |
| with: | |
| name: results ${{ matrix.name }} | |
| path: | | |
| result*.json | |
| result*.csv | |
| - name: Finish benchmarking | |
| run: echo "Complete" | |
| if: success() || failure() |