WEB3-1383 - aggregator app test

.env.local.aggregator

@@ -0,0 +1,6 @@
+PARALLEL=true
+DEV_MODE=true
+LOCAL_NODE=true
+RPC_URL=http://localhost:9944
+WEBSOCKET=ws://localhost:9944
+SEED_PHRASE_1=//Alice

.gitignore

@@ -17,6 +17,9 @@ src/e2e-tests/services/zkVerify
 src/e2e-tests/services/nh-attestation-bot
 src/e2e-tests/services/zkv-attestation-contracts
 src/e2e-tests/*.tar
+src/aggregator-tests/services/zkVerify
+src/aggregator-tests/services/zkverify-aggregation
+src/aggregator-tests/*.tar
 src/proof-generator/fflonk/circuit/data
 src/proof-generator/groth16/circuit/data
 src/proof-generator/boojum/data

jest.config.aggregator.js

@@ -0,0 +1,22 @@
+const path = require('path');
+
+const config = {
+  rootDir: path.resolve(__dirname, 'src'),
+  testTimeout: 120000,
+  roots: ['<rootDir>/aggregator-tests'],
+  testMatch: [
+    '**/?(*.)+(spec|test).[tj]s?(x)',
+  ],
+  moduleFileExtensions: ['js', 'json', 'jsx', 'ts', 'tsx', 'node'],
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest',
+  },
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  reporters: ["default"],
+  testPathIgnorePatterns: [
+    '<rootDir>/aggregator-tests/services/'
+  ],
+};
+
+module.exports = config;

package.json

@@ -6,6 +6,7 @@
   "license": "ISC",
   "scripts": {
     "test:e2e": "cp .env.local.e2e .env && jest --config=jest.config.e2e.js --forceExit --verbose",
+    "test:aggregator": "cp .env.local.aggregator .env && jest --config=jest.config.aggregator.js --forceExit --verbose",
     "test:rpc:local": "cp .env.local.rpc .env && jest --config=jest.config.local.rpc.js --verbose",
     "test:rpc:testnet": "cp .env.testnet.rpc .env && jest --config=jest.config.testnet.rpc.js --forceExit --verbose",
     "generate:single:proof": "cp .env.generator .env && npx ts-node src/send-proof/index.ts",

src/aggregator-tests/docker-compose.yml

@@ -0,0 +1,58 @@
+services:
+
+  local_node:
+    image: horizenlabs/zkverify:${ZKVERIFY_IMAGE_TAG}
+    container_name: "${NODE_NAME}"
+    hostname: "${NODE_NAME}"
+    env_file:
+      - ./services/zkVerify/docker/resources/envs/.env.local
+    ports:
+      - "${NODE_NET_RPC_WS_PORT}:${NODE_NET_RPC_WS_PORT}"
+      - "${NODE_NET_P2P_PORT}:${NODE_NET_P2P_PORT}"
+    expose:
+      - "${NODE_NET_RPC_WS_PORT}"
+    healthcheck:
+      test:
+        [
+          CMD,
+          curl,
+          -H,
+          "Content-Type: application/json",
+          "-d",
+          '{"jsonrpc":"2.0","method":"system_health","params":[],"id":1}',
+          -f,
+          "http://local_node:9944",
+        ]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+
+  node_alice:
+    image: horizenlabs/zkverify:${ZKVERIFY_IMAGE_TAG}
+    env_file:
+      - ./services/zkVerify/docker/resources/envs/.env.alice
+
+  node_bob:
+    image: horizenlabs/zkverify:${ZKVERIFY_IMAGE_TAG}
+    env_file:
+      - ./services/zkVerify/docker/resources/envs/.env.bob
+    volumes:
+      - ./services/zkVerify/docker/resources/deploy:/data/config
+
+  aggregation-app:
+    build:
+      context: ./services/zkverify-aggregation
+      dockerfile: Dockerfile
+    environment:
+      - AGGREGATOR_PROVIDER_WS=${AGGREGATOR_PROVIDER_WS}
+      - AGGREGATOR_SENDER_CREDENTIALS=${AGGREGATOR_SENDER_CREDENTIALS}
+    env_file:
+      - .env
+    depends_on:
+      local_node:
+        condition: service_healthy
+      node_alice:
+        condition: service_started
+      node_bob:
+        condition: service_started

src/aggregator-tests/setup.sh

@@ -0,0 +1,175 @@
+#!/bin/bash
+set -eou pipefail
+
+# Default values
+rebuild=0
+fetch_latest=0
+zkverify_version="main"
+zkverify_aggregation_branch="main"
+docker_image_tag="latest"
+
+# Command-line options
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        --rebuild) rebuild=1 ;;
+        --fetch-latest) fetch_latest=1 ;;
+        --zkverify-version) zkverify_version="$2"; shift ;;
+        --zkverify-aggregation) zkverify_aggregation_branch="$2"; shift ;;
+        --docker-image-tag) docker_image_tag="$2"; shift ;;
+        *) echo "Unknown parameter passed: $1"; exit 1 ;;
+    esac
+    shift
+done
+
+# Repositories
+repo_names=("zkVerify" "zkverify-aggregation")
+repo_urls=("https://github.com/HorizenLabs/zkVerify.git" "https://github.com/zkVerify/zkverify-aggregation.git")
+repo_branches=("$zkverify_version" "$zkverify_aggregation_branch")
+repo_count=${#repo_names[@]}
+
+function check_docker_hub_image() {
+    local image_name=$1
+    local tag=$2
+    if curl --silent --fail "https://hub.docker.com/v2/repositories/${image_name}/tags/${tag}/" > /dev/null; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+function check_local_image() {
+    local image_name=$1
+    local tag=$2
+
+    image_id=$(docker images -q ${image_name}:${tag})
+
+    if [[ -n "$image_id" ]]; then
+        echo "Image ${image_name}:${tag} exists locally."
+        return 0
+    else
+        echo "Image ${image_name}:${tag} does not exist locally."
+        return 1
+    fi
+}
+
+function build_zkverify_image() {
+    echo "Configuring and bootstrapping zkVerify..."
+    cd services/zkVerify || exit 1
+
+    # Source config
+    if [ -f "cfg" ]; then
+        source cfg
+    else
+        echo "Configuration file not found at the top level, check the path and filename."
+        exit 1
+    fi
+
+    if [ -f "docker/dockerfiles/zkv-node.Dockerfile" ]; then
+        echo "Building zkVerify image with tag: ${docker_image_tag}"
+        docker build -f docker/dockerfiles/zkv-node.Dockerfile -t "${image_name}:${docker_image_tag}" .
+        echo "zkVerify image tagged as ${image_name}:${docker_image_tag}"
+        echo "zkVerify is set up and ready."
+    else
+        echo "zkv-node.Dockerfile not found in 'docker/dockerfiles/', check the path and filename."
+        exit 1
+    fi
+
+    cd ../..
+}
+
+# Check if running in GitHub Actions
+if [ "${GITHUB_ACTIONS:-}" = "true" ]; then
+    echo "Running in GitHub Actions."
+    if [ -z "${GH_TOKEN:-}" ]; then
+        echo "Error: GH_TOKEN is not set. Please set it as a secret in your GitHub Actions workflow."
+        exit 1
+    fi
+    auth_prefix="https://${GH_TOKEN}@"
+else
+    echo "Running locally. Using default authentication."
+    auth_prefix=""
+fi
+
+# Clone each repository into the services directory or fetch latest updates
+for ((i=0; i<repo_count; i++)); do
+    repo=${repo_names[$i]}
+    repo_url=${repo_urls[$i]}
+    repo_branch_or_tag=${repo_branches[$i]}
+    target_dir="./services/$repo"
+
+    if [ ! -d "$target_dir" ]; then
+        echo "Directory $target_dir does not exist. Cloning..."
+        if [ -n "$auth_prefix" ]; then
+            # Running in GitHub Actions
+            git clone --no-single-branch "${auth_prefix}${repo_url#https://}" "$target_dir"
+        else
+            # Running locally
+            git clone --no-single-branch "$repo_url" "$target_dir"
+        fi
+        echo "Repository $repo cloned successfully."
+    else
+        echo "Directory $target_dir already exists."
+        if [ "$fetch_latest" -eq 1 ]; then
+            echo "Fetching latest for $repo..."
+            (cd "$target_dir" && git fetch --all)
+        else
+            echo "Skipping update for $repo."
+        fi
+    fi
+
+    # Checkout the specific branch, tag, or commit hash
+    (
+        cd "$target_dir"
+
+        # Attempt to checkout as branch or tag
+        if git rev-parse --verify "$repo_branch_or_tag" >/dev/null 2>&1; then
+            git checkout "$repo_branch_or_tag"
+            echo "Checked out ${repo_branch_or_tag} for ${repo}."
+        else
+            # Assume it's a commit hash
+            echo "Attempting to fetch and checkout commit hash ${repo_branch_or_tag}..."
+            git fetch origin "$repo_branch_or_tag" || { echo "Failed to fetch commit ${repo_branch_or_tag}"; exit 1; }
+            git checkout "$repo_branch_or_tag" || { echo "Failed to checkout commit ${repo_branch_or_tag}"; exit 1; }
+            echo "Checked out commit ${repo_branch_or_tag} for ${repo}."
+        fi
+    )
+done
+
+# If rebuild = 1, build Docker image
+# If rebuild = 0, check Docker hub and local Docker images. Build a docker image if not available
+image_name="horizenlabs/zkverify"
+
+if [[ "$rebuild" -eq 1 ]]; then
+    containers=$(docker ps -a -q --filter ancestor="${image_name}:${docker_image_tag}")
+    if [ -n "$containers" ]; then
+        echo "Stopping and removing containers using the image ${image_name}:${docker_image_tag}..."
+        docker stop $containers
+        docker rm -f $containers
+    fi
+
+    if docker images -q "${image_name}:${docker_image_tag}"; then
+        echo "Removing image ${image_name}:${docker_image_tag}..."
+        docker rmi -f "${image_name}:${docker_image_tag}"
+    fi
+
+    build_zkverify_image
+else
+    if check_local_image "$image_name" "$docker_image_tag"; then
+        echo "Using locally available image ${image_name}:${docker_image_tag}."
+    elif check_docker_hub_image "$image_name" "$docker_image_tag"; then
+        echo "Using Docker Hub image ${image_name}:${docker_image_tag}."
+        docker pull "${image_name}:${docker_image_tag}"
+    else
+        echo "Image ${image_name}:${docker_image_tag} not found locally or on Docker Hub. Building image..."
+        build_zkverify_image
+    fi
+fi
+
+check_local_image "$image_name" "$docker_image_tag"
+
+# Save the Docker image as a tarball
+current_dir="$(dirname "$(realpath "$0")")"
+echo "Saving image ${image_name}:${docker_image_tag} to ${current_dir}/zkverify-image.tar"
+docker save "${image_name}:${docker_image_tag}" -o "${current_dir}/zkverify-image.tar"
+
+echo "Setup completed."

src/aggregator-tests/tests/index.test.ts

@@ -0,0 +1,54 @@
+import { executeRegisterDomain } from '../../utils/aggregation/register-domains';
+import { checkDomain, checkMultipleDomains, getMaxAggregationSize } from '../../utils/aggregation/check-domains';
+import { generateAndVerifyProofs } from '../../utils/aggregation/send-multiple-proofs';
+import { setTimeout } from 'timers/promises';
+
+jest.setTimeout(300000);
+
+const wsProviderUrl = 'ws://127.0.0.1:9944';
+const signingUser = '//Alice';
+
+const proofType = 'fflonk';
+
+const firstDomainId = 0;
+const secondDomainId = 1;
+
+let aggregationSize = 2;
+let queueSize = 10;
+let numberOfAggregations = 2;
+
+describe('zkVerify proof aggregation tests', () => {
+    test('check the proofs are correctly aggregated by the aggregator application', async () => {
+
+        const domainIds = [0, 1];
+
+        try {
+            await checkMultipleDomains(wsProviderUrl, domainIds);
+            console.log("All domains are already present. Proceeding...");
+        } catch {
+            console.log("Domain check failed. Register the domain IDs...");
+            await executeRegisterDomain(wsProviderUrl, signingUser, aggregationSize, queueSize, numberOfAggregations);
+        }
+
+        let nextIdFirstDomain = await checkDomain(wsProviderUrl, firstDomainId);
+        let nextIdSecondDomain = await checkDomain(wsProviderUrl, secondDomainId);
+
+        let maxAggregationSizeFirstDomain = await getMaxAggregationSize(wsProviderUrl, firstDomainId);
+        await generateAndVerifyProofs(wsProviderUrl, signingUser, proofType, maxAggregationSizeFirstDomain, firstDomainId)
+            .then(() => console.log('Proof generation and verification completed.'))
+            .catch(error => console.error('Error:', error));
+
+        let maxAggregationSizeSecondDomain = await getMaxAggregationSize(wsProviderUrl, secondDomainId);
+        await generateAndVerifyProofs(wsProviderUrl, signingUser, proofType, maxAggregationSizeSecondDomain, secondDomainId)
+            .then(() => console.log('Proof generation and verification completed.'))
+            .catch(error => console.error('Error:', error));
+
+        await setTimeout(10000);
+
+        await checkDomain(wsProviderUrl, firstDomainId, nextIdFirstDomain+1);
+        await checkDomain(wsProviderUrl, secondDomainId, nextIdSecondDomain+1);
+
+    });
+
+
+});