Benchmark (all) #186
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| run-name: "Benchmark (${{ github.event.inputs.datastore == 'custom' && github.event.inputs.custom || github.event.inputs.datastore }})" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| datastore: | |
| type: choice | |
| description: Datastore | |
| required: true | |
| options: | |
| - all | |
| - arangodb | |
| - cassandra | |
| - dragonfly | |
| - dry | |
| - fjall | |
| - keydb | |
| - lmdb | |
| - map | |
| - mongodb | |
| - mysql | |
| - neo4j | |
| - postgres | |
| - redb | |
| - redis | |
| - rocksdb | |
| - scylladb | |
| - sqlite | |
| - surrealdb | |
| - surrealdb-memory | |
| - surrealdb-rocksdb | |
| - surrealdb-surrealkv | |
| - surrealdb-embedded-memory | |
| - surrealdb-embedded-rocksdb | |
| - surrealdb-embedded-surrealkv | |
| - surrealkv | |
| - custom | |
| custom: | |
| type: string | |
| description: Custom choice | |
| required: false | |
| samples: | |
| type: string | |
| description: Number of samples | |
| required: true | |
| default: "5000000" | |
| clients: | |
| type: string | |
| description: Number of clients | |
| required: true | |
| default: "128" | |
| threads: | |
| type: string | |
| description: Number of threads | |
| required: true | |
| default: "48" | |
| key-type: | |
| type: choice | |
| description: Primary key type | |
| required: true | |
| default: "string26" | |
| options: | |
| - "integer" | |
| - "string26" | |
| - "string90" | |
| - "string250" | |
| - "string506" | |
| value: | |
| type: string | |
| description: Item content | |
| required: true | |
| default: '{ "text": "text:50", "integer": "int", "nested": { "text": "text:1000", "array": [ "string:50", "string:50", "string:50", "string:50", "string:50" ] } }' | |
| sync: | |
| type: choice | |
| description: Acknowledge disk writes | |
| required: true | |
| default: "false" | |
| options: | |
| - "true" | |
| - "false" | |
| cooldown: | |
| type: string | |
| description: Cooldown in minutes | |
| required: true | |
| default: "15" | |
| timeout: | |
| type: string | |
| description: Timeout in minutes | |
| required: true | |
| default: "300" | |
| concurrency: | |
| # Use github.run_id on main branch | |
| # Use github.event.pull_request.number on pull requests, so it's unique per pull request | |
| # Use github.ref on other branches, so it's unique per branch | |
| group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| build: | |
| name: Build crud-bench | |
| runs-on: [runner-amd64-4xlarge] | |
| steps: | |
| - name: Install stable toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Checkout sources | |
| uses: actions/checkout@v4 | |
| - name: Setup cache | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| cache-on-failure: true | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - name: Build benchmark | |
| run: cargo build --release --target x86_64-unknown-linux-gnu | |
| - name: Store artifacts | |
| run: cp target/x86_64-unknown-linux-gnu/release/crud-bench crud-bench | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: crud-bench | |
| path: crud-bench | |
| benchmark: | |
| name: Benchmark ${{ matrix.description }} | |
| needs: build | |
| runs-on: [runner-benchmarking] | |
| continue-on-error: true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # ArangoDB | |
| - name: arangodb | |
| database: arangodb | |
| enabled: true | |
| description: ArangoDB | |
| DOCKER_PRE_ARGS: "-v /data/crud-bench:/var/lib/arangodb3" | |
| # Cassandra | |
| - name: cassandra | |
| database: cassandra | |
| enabled: false | |
| description: Cassandra | |
| skipped: Cassandra benchmark not yet implemented | |
| # Dragonfly | |
| - name: dragonfly | |
| database: dragonfly | |
| enabled: true | |
| description: Dragonfly | |
| # Dry | |
| - name: dry | |
| database: dry | |
| enabled: true | |
| description: Dry | |
| # Fjall | |
| - name: fjall | |
| database: fjall | |
| enabled: true | |
| description: Fjall | |
| # KeyDB | |
| - name: keydb | |
| database: keydb | |
| enabled: true | |
| description: KeyDB | |
| # LMDB | |
| - name: lmdb | |
| database: lmdb | |
| enabled: true | |
| description: LMDB | |
| # Map | |
| - name: map | |
| database: map | |
| enabled: true | |
| description: Map | |
| # MongoDB | |
| - name: mongodb | |
| database: mongodb | |
| enabled: true | |
| description: MongoDB | |
| DOCKER_PRE_ARGS: "-v /data/crud-bench:/data/db" | |
| # MySQL | |
| - name: mysql | |
| database: mysql | |
| enabled: true | |
| description: MySQL | |
| DOCKER_PRE_ARGS: "-v /data/crud-bench:/var/lib/mysql" | |
| # Neo4j | |
| - name: neo4j | |
| database: neo4j | |
| enabled: true | |
| description: Neo4j | |
| DOCKER_PRE_ARGS: "-v /data/crud-bench:/data" | |
| # Postgres | |
| - name: postgres | |
| database: postgres | |
| enabled: true | |
| description: Postgres | |
| DOCKER_PRE_ARGS: "-v /data/crud-bench:/var/lib/postgresql/data" | |
| # Redb | |
| - name: redb | |
| database: redb | |
| enabled: false | |
| description: ReDB | |
| skipped: ReDB benchmark skipped due to excessive benchmark time | |
| # Redis | |
| - name: redis | |
| database: redis | |
| enabled: true | |
| description: Redis | |
| # RocksDB | |
| - name: rocksdb | |
| database: rocksdb | |
| enabled: true | |
| description: RocksDB | |
| # Scylladb | |
| - name: scylladb | |
| database: scylladb | |
| enabled: false | |
| description: ScyllaDB | |
| skipped: ScyllaDB benchmark not yet implemented | |
| # SQLite | |
| - name: sqlite | |
| database: sqlite | |
| enabled: true | |
| description: SQLite | |
| # SurrealDB + Memory | |
| - name: surrealdb-memory | |
| database: surrealdb-memory | |
| enabled: true | |
| description: SurrealDB with in-memory storage | |
| # SurrealDB + RocksDB | |
| - name: surrealdb-rocksdb | |
| database: surrealdb-rocksdb | |
| enabled: true | |
| description: SurrealDB with RocksDB storage | |
| DOCKER_PRE_ARGS: "-v /data/crud-bench:/data -e SURREAL_ROCKSDB_BACKGROUND_FLUSH=true" | |
| # SurrealDB + SurrealKV | |
| - name: surrealdb-surrealkv | |
| database: surrealdb-surrealkv | |
| enabled: true | |
| description: SurrealDB with SurrealKV storage | |
| DOCKER_PRE_ARGS: "-v /data/crud-bench:/data" | |
| # SurrealDB Memory Engine | |
| - name: surrealdb-embedded-memory | |
| database: surrealdb | |
| enabled: true | |
| endpoint: -e memory | |
| description: SurrealDB embedded with in-memory storage | |
| # SurrealDB RocksDB Engine | |
| - name: surrealdb-embedded-rocksdb | |
| database: surrealdb | |
| enabled: true | |
| endpoint: -e rocksdb:/data/crud-bench | |
| description: SurrealDB embedded with RocksDB storage | |
| # SurrealDB SurrealKV Engine | |
| - name: surrealdb-embedded-surrealkv | |
| database: surrealdb | |
| enabled: true | |
| endpoint: -e surrealkv:/data/crud-bench | |
| description: SurrealDB embedded with SurrealKV storage | |
| # SurrealKV | |
| - name: surrealkv | |
| database: surrealkv | |
| enabled: true | |
| description: SurrealKV | |
| steps: | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: ${{ github.workspace }}/artifacts | |
| merge-multiple: true | |
| - name: Set file permissions | |
| run: chmod +x ${{ github.workspace }}/artifacts/crud-bench | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USER }} | |
| password: ${{ secrets.DOCKER_TOKEN }} | |
| - name: ${{ matrix.skipped || 'Benchmark processing' }} | |
| if: ${{ !matrix.enabled }} | |
| run: echo "${{ matrix.skipped }}" | |
| - name: Clean up environment | |
| if: ${{ matrix.enabled && ( github.event.inputs.datastore == 'all' || contains(matrix.name, github.event.inputs.datastore) || ( github.event.inputs.datastore == 'custom' && contains(github.event.inputs.custom, matrix.name) ) ) && (success() || failure()) }} | |
| run: | | |
| docker container kill crud-bench &>/dev/null || true | |
| docker container prune --force | |
| docker volume prune --force | |
| docker system prune --force | |
| rm -rf /data/crud-bench | |
| mkdir -p /data/crud-bench | |
| chmod 777 /data/crud-bench | |
| - name: Optimise system | |
| if: ${{ matrix.enabled && ( github.event.inputs.datastore == 'all' || contains(matrix.name, github.event.inputs.datastore) || ( github.event.inputs.datastore == 'custom' && contains(github.event.inputs.custom, matrix.name) ) ) && (success() || failure()) }} | |
| run: | | |
| # Disable services | |
| sudo systemctl stop unattended-upgrades | |
| # Flush disk writes | |
| sync | |
| # Clear page tables | |
| echo 1 | sudo tee /proc/sys/vm/compact_memory | |
| # Drop memory caches | |
| echo 3 | sudo tee /proc/sys/vm/drop_caches | |
| # Disable Transparent Hugh Pages | |
| echo never | sudo tee /sys/kernel/mm/transparent_hugepage/enabled | |
| # Disable swap memory | |
| sudo swapoff -a | |
| # Increase max limits | |
| ulimit -n 65536 | |
| ulimit -u unlimited | |
| ulimit -l unlimited | |
| - name: Wait for system cool down | |
| if: ${{ matrix.enabled && ( github.event.inputs.datastore == 'all' || contains(matrix.name, github.event.inputs.datastore) || ( github.event.inputs.datastore == 'custom' && contains(github.event.inputs.custom, matrix.name) ) ) && (success() || failure()) }} | |
| run: sleep ${{ github.event.inputs.cooldown }}m | |
| - name: System Information | |
| run: | | |
| echo "=== Environment Variables ===" | |
| env | |
| echo "=== Kernel & OS Info ===" | |
| uname -a | |
| echo "=== CPU Details (lscpu) ===" | |
| lscpu | |
| echo "=== First 50 lines of /proc/cpuinfo ===" | |
| head -n 50 /proc/cpuinfo | |
| echo "=== First 50 lines of /proc/meminfo ===" | |
| head -n 50 /proc/meminfo | |
| echo "=== Cgroup Information (/proc/self/cgroup) ===" | |
| cat /proc/self/cgroup | |
| - name: Run benchmarks (${{ github.event.inputs.samples }} samples / ${{ github.event.inputs.clients }} clients / ${{ github.event.inputs.threads }} threads / key ${{ github.event.inputs.key-type }} / random) | |
| continue-on-error: true | |
| timeout-minutes: ${{ fromJSON(github.event.inputs.timeout) }} | |
| if: ${{ matrix.enabled && ( github.event.inputs.datastore == 'all' || contains(matrix.name, github.event.inputs.datastore) || ( github.event.inputs.datastore == 'custom' && contains(github.event.inputs.custom, matrix.name) ) ) && (success() || failure()) }} | |
| run: sudo taskset -c 0-$(($(nproc) - 1)) sudo nice -n -20 sudo ionice -c 1 -n 0 sudo ${{ github.workspace }}/artifacts/crud-bench --privileged ${{ github.event.inputs.sync == 'true' && '--sync' || '' }} -d ${{ matrix.database }} ${{ matrix.endpoint || '' }} -s ${{ github.event.inputs.samples }} -c ${{ github.event.inputs.clients }} -t ${{ github.event.inputs.threads }} -k ${{ github.event.inputs.key-type }} -n benchmark -r | |
| env: | |
| CRUD_BENCH_LMDB_DATABASE_SIZE: 32212254720 # 30 GiB | |
| CRUD_BENCH_VALUE: ${{ github.event.inputs.value }} | |
| DOCKER_PRE_ARGS: ${{ matrix.DOCKER_PRE_ARGS || '' }} | |
| DOCKER_POST_ARGS: ${{ matrix.DOCKER_POST_ARGS || '' }} | |
| - name: Normalise system | |
| if: ${{ matrix.enabled && ( github.event.inputs.datastore == 'all' || contains(matrix.name, github.event.inputs.datastore) || ( github.event.inputs.datastore == 'custom' && contains(github.event.inputs.custom, matrix.name) ) ) && (success() || failure()) }} | |
| run: | | |
| # Enable services | |
| sudo systemctl start unattended-upgrades | |
| # Flush disk writes | |
| sync | |
| # Clear page tables | |
| echo 1 | sudo tee /proc/sys/vm/compact_memory | |
| # Drop memory caches | |
| echo 3 | sudo tee /proc/sys/vm/drop_caches | |
| # Enable Transparent Hugh Pages | |
| echo always | sudo tee /sys/kernel/mm/transparent_hugepage/enabled | |
| - name: Upload result artifacts | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ matrix.enabled && ( github.event.inputs.datastore == 'all' || contains(matrix.name, github.event.inputs.datastore) ) && (success() || failure()) }} | |
| with: | |
| name: results ${{ matrix.name }} | |
| path: | | |
| result*.json | |
| result*.csv | |
| - name: Finish benchmarking | |
| run: echo "Complete" | |
| if: success() || failure() |