openapi.yaml

openapi: 3.1.0
info:
  title: Eval Hub
  description: API REST server for evaluation backend orchestration
  version: 0.3.0
  contact:
    name: API Support
  license:
    name: Apache 2.0
    url: https://www.apache.org/licenses/LICENSE-2.0
servers:
  - url: http://localhost:8080
    description: Local development server
security: []
tags:
  - name: Evaluations
    description: Evaluation job management endpoints
  - name: Collections
    description: Benchmark collection management endpoints
  - name: Providers
    description: Evaluation provider endpoints
  - name: Health
    description: Health check endpoints
paths:
  /api/v1/health:
    get:
      summary: Health Check
      description: Health check endpoint.
      operationId: get_health
      tags:
        - Health
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HealthResponse'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
  /api/v1/evaluations/jobs:
    post:
      tags:
        - Evaluations
      summary: Create Evaluation
      description: Create and execute evaluation request using the simplified benchmark schema.
      operationId: post_evaluations_jobs
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/EvaluationJobConfig'
      responses:
        '202':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/EvaluationJobResource'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
    get:
      tags:
        - Evaluations
      summary: List Evaluations
      description: List all evaluation requests.
      operationId: get_evaluations_jobs
      parameters:
        - name: limit
          in: query
          required: false
          schema:
            type: integer
            maximum: 100
            minimum: 1
            description: Maximum number of evaluations to return
            default: 50
            title: Limit
          description: Maximum number of evaluations to return
        - name: offset
          in: query
          required: false
          schema:
            type: integer
            minimum: 0
            description: Offset for pagination
            default: 0
            title: Offset
          description: Offset for pagination
        - name: status
          in: query
          required: false
          schema:
            type: string
            description: Filter by status
            title: Status Filter
          description: Filter by status
        - name: name
          in: query
          required: false
          schema:
            type: string
            title: Name
          description: Name to search for
        - name: tags
          in: query
          required: false
          schema:
            type: string
            title: Tags
          description: Tags to search for
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/EvaluationJobResourceList'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
  /api/v1/evaluations/jobs/{id}:
    get:
      tags:
        - Evaluations
      summary: Get Evaluation
      description: Returns the evaluation job resource with the current status and results.
      operationId: get_evaluations_jobs_id
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
            title: Id
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/EvaluationJobResource'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
    delete:
      tags:
        - Evaluations
      summary: Cancel Evaluation
      description: Cancel a running evaluation.
      operationId: delete_evaluations_jobs_id
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
            title: Id
        - name: hard_delete
          in: query
          required: false
          schema:
            type: boolean
            description: If `true`, delete the evaluation job permanently so that `GET /api/v1/evaluations/jobs/{id}` will return a 404.
            default: false
            title: Hard Delete
      responses:
        '204':
          description: Successful Response
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
        '409':
          $ref: '#/components/responses/Conflict'
  /api/v1/evaluations/providers:
    get:
      tags:
        - Providers
      summary: List Providers
      description: List all registered evaluation providers.
      operationId: list_providers
      parameters:
        - name: limit
          in: query
          required: false
          schema:
            type: integer
            maximum: 100
            minimum: 1
            description: Maximum number of providers to return
            default: 50
            title: Limit
          description: Maximum number of providers to return
        - name: offset
          in: query
          required: false
          schema:
            type: integer
            minimum: 0
            description: Offset for pagination
            default: 0
            title: Offset
          description: Offset for pagination
        - name: benchmarks
          in: query
          required: false
          schema:
            type: boolean
            default: true
            description: Set to false to not include benchmarks supported by this provider in the response
            title: Benchmarks
          description: Include or exclude benchmarks supported by this provider in the response
        - name: name
          in: query
          required: false
          schema:
            type: string
            title: Name
          description: Name to search for
        - name: tags
          in: query
          required: false
          schema:
            type: string
            title: Tags
          description: Tags to search for
        - name: scope
          in: query
          required: false
          schema:
            type: string
            enum:
              - system
              - tenant
            title: Scope of providers
          description: |
            Set to `system` to get only system defined providers, or `tenant` to get only user defined providers. If `scope` is not provided, both system and user defined providers will be returned.
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ProviderResourceList'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
    post:
      tags:
        - Providers
      summary: Create a new provider scoped to the current tenant (Bring Your Own Provider)
      description: Create a new provider scoped to the current tenant (Bring Your Own Provider)
      operationId: post_providers
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/ProviderConfig'
        required: true
      responses:
        '201':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ProviderResource'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
  /api/v1/evaluations/providers/{id}:
    get:
      tags:
        - Providers
      summary: Get Provider
      description: Get a provider by ID.
      operationId: get_providers_id
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
            description: Provider ID
            title: Provider Id
          description: Provider ID
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ProviderResource'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
    put:
      tags:
        - Providers
      summary: Update Provider
      description: Update an existing provider.
      operationId: put_providers_id
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
            description: Provider ID
            title: Provider Id
          description: Provider ID
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/ProviderConfig'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ProviderResource'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
    patch:
      tags:
        - Providers
      summary: Patch Provider
      description: Partially update an existing provider.
      operationId: patch_providers_id
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
            title: Provider Id
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: array
              title: Json Patch
              description: JSON Patch operation
              items:
                $ref: '#/components/schemas/PatchOperation'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ProviderResource'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
    delete:
      tags:
        - Providers
      summary: Delete Provider
      description: Delete provider by ID.
      operationId: delete_providers_id
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
            description: Provider ID
            title: Provider Id
          description: Provider ID
      responses:
        '204':
          description: Successful Response
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
  /api/v1/evaluations/collections:
    get:
      tags:
        - Collections
      summary: List Collections
      description: List all benchmark collections.
      operationId: get_collections
      parameters:
        - name: limit
          in: query
          required: false
          schema:
            type: integer
            maximum: 100
            minimum: 1
            description: Maximum number of collections to return
            default: 50
            title: Limit
          description: Maximum number of collections to return
        - name: offset
          in: query
          required: false
          schema:
            type: integer
            minimum: 0
            description: Offset for pagination
            default: 0
            title: Offset
          description: Offset for pagination
        - name: name
          in: query
          required: false
          schema:
            type: string
            title: Name
          description: Name to search for
        - name: category
          in: query
          required: false
          schema:
            type: string
            title: Category
          description: Category to search for
        - name: tags
          in: query
          required: false
          schema:
            type: string
            title: Tags
          description: Tags to search for
        - name: scope
          in: query
          required: false
          schema:
            type: string
            enum:
              - system
              - tenant
            title: Scope of collections
          description: |
            Set to `system` to get only system defined collections, or `tenant` to get only user defined collections. If `scope` is not provided, both system and user defined collections will be returned.
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/CollectionResourceList'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
    post:
      tags:
        - Collections
      summary: Create Collection
      description: Create a new collection.
      operationId: post_collections
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CollectionConfig'
        required: true
      responses:
        '201':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/CollectionResource'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
  /api/v1/evaluations/collections/{id}:
    get:
      tags:
        - Collections
      summary: Get Collection
      description: Get details of a specific collection.
      operationId: get_collections_id
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
            title: Collection Id
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/CollectionResource'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
    put:
      tags:
        - Collections
      summary: Update Collection
      description: Update an existing collection.
      operationId: put_collections_id
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
            title: Collection Id
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CollectionConfig'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/CollectionResource'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
    patch:
      tags:
        - Collections
      summary: Patch Collection
      description: Partially update an existing collection.
      operationId: patch_collections_id
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
            title: Collection Id
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: array
              title: Json Patch
              description: JSON Patch operation
              items:
                $ref: '#/components/schemas/PatchOperation'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/CollectionResource'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
    delete:
      tags:
        - Collections
      summary: Delete Collection
      description: Delete a collection.
      operationId: delete_collections_id
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
            title: Collection Id
      responses:
        '204':
          description: Success
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
components:
  schemas:
    HealthResponse:
      type: object
      description: Health check response
      properties:
        status:
          type: string
          description: Overall status (e.g. healthy, degraded)
        version:
          type: string
          description: Service version
        timestamp:
          type: string
          format: date-time
          description: Server timestamp
        components:
          type: object
          additionalProperties:
            type: object
            additionalProperties: true
          description: Per-component health
        uptime:
          type: integer
          description: Uptime in nanoseconds (or use string for duration)
        active_evaluations:
          type: integer
          description: Number of active evaluations
    Error:
      type: object
      description: Error response
      properties:
        message_code:
          type: string
          description: Machine-readable error code
        message:
          type: string
          description: Human-readable message
        trace:
          type: string
          description: Request trace or debug info
      required:
        - message_code
        - message
    HRef:
      type: object
      description: Hypermedia reference
      properties:
        href:
          type: string
          description: URI reference
    Page:
      type: object
      description: Generic pagination schema
      properties:
        first:
          $ref: '#/components/schemas/HRef'
          description: Link to first page
        next:
          $ref: '#/components/schemas/HRef'
          description: Link to next page (omit if none)
        limit:
          type: integer
          description: Page size
        total_count:
          type: integer
          description: Total number of items
    Resource:
      type: object
      description: Base resource fields
      properties:
        id:
          type: string
          description: Unique identifier
        tenant:
          type: string
          description: Tenant for scoping
        created_at:
          type: string
          format: date-time
          description: Creation timestamp
        updated_at:
          type: string
          format: date-time
          description: Last update timestamp
        owner:
          type: string
          description: Owner of the resource
    EvaluationResource:
      type: object
      description: Evaluation resource (base fields for an evaluation)
      allOf:
        - $ref: '#/components/schemas/Resource'
        - type: object
          properties:
            mlflow_experiment_id:
              type: string
              description: MLFlow experiment ID
    OverallState:
      type: string
      enum:
        - pending
        - running
        - completed
        - failed
        - cancelled
        - partially_failed
      description: Overall evaluation job state
    MessageInfo:
      type: object
      description: Message from a downstream service
      properties:
        message:
          type: string
          description: Human-readable message
        message_code:
          type: string
          description: Machine-readable code
    EvaluationJobState:
      type: object
      description: Overall evaluation job state and message
      required:
        - state
        - message
      properties:
        state:
          $ref: '#/components/schemas/OverallState'
        message:
          $ref: '#/components/schemas/MessageInfo'
    State:
      type: string
      enum:
        - pending
        - running
        - completed
        - failed
        - cancelled
      description: Evaluation or benchmark state
    BenchmarkStatus:
      type: object
      description: Status of an individual benchmark in an evaluation
      properties:
        provider_id:
          type: string
          description: Provider ID
        id:
          type: string
          description: Benchmark ID
        benchmark_index:
          type: integer
          description: Benchmark index in the evaluation job request
        status:
          $ref: '#/components/schemas/State'
        error_message:
          $ref: '#/components/schemas/MessageInfo'
        started_at:
          type: string
          format: date-time
          description: RFC3339 start time
        completed_at:
          type: string
          format: date-time
          description: RFC3339 completion time
    EvaluationJobStatus:
      type: object
      description: Current status of an evaluation job
      allOf:
        - $ref: '#/components/schemas/EvaluationJobState'
        - type: object
          properties:
            benchmarks:
              type: array
              items:
                $ref: '#/components/schemas/BenchmarkStatus'
              description: Per-benchmark status
    BenchmarkTest:
      type: object
      description: The test result of a single benchmark run
      properties:
        primary_score:
          type: number
          format: float
          description: Primary score value
        threshold:
          type: number
          format: float
          description: Threshold value
        pass:
          type: boolean
          description: Whether the benchmark passed
    BenchmarkResult:
      type: object
      description: Result of a single benchmark run
      properties:
        id:
          type: string
          description: Benchmark ID
        provider_id:
          type: string
          description: Provider ID
        benchmark_index:
          type: integer
          description: Benchmark index in the evaluation job request
        metrics:
          type: object
          additionalProperties: true
          description: Metric name to value
        artifacts:
          type: object
          additionalProperties: true
          description: Artifact key to location/info
        mlflow_run_id:
          type: string
          description: MLFlow run ID
        logs_path:
          type: string
          description: Path to logs
        test:
          $ref: '#/components/schemas/BenchmarkTest'
          description: Test result
    EvaluationTest:
      type: object
      description: The test result for an evaluation
      properties:
        score:
          type: number
          format: float
          description: Evaluation score value
        threshold:
          type: number
          format: float
          description: Threshold value
        pass:
          type: boolean
          description: Whether the entire evaluation passed
    EvaluationJobResults:
      type: object
      description: Results section for an evaluation job
      properties:
        benchmarks:
          type: array
          items:
            $ref: '#/components/schemas/BenchmarkResult'
          description: Per-benchmark results
        mlflow_experiment_url:
          type: string
          description: URL to MLFlow experiment
        test:
          $ref: '#/components/schemas/EvaluationTest'
          description: Test result
    ModelAuth:
      type: object
      description: The model authentication configuration
      properties:
        secret_ref:
          type: string
          description: The reference to the secret containing the model authentication credentials
    ModelRef:
      type: object
      description: Model specification for evaluation requests
      required:
        - url
        - name
      properties:
        url:
          type: string
          description: Model URL
        name:
          type: string
          description: Model name
        parameters:
          type: object
          additionalProperties: true
          description: Model specific parameters
        auth:
          $ref: '#/components/schemas/ModelAuth'
          description: The model authentication configuration
    EvaluationJobConfigCommon1:
      type: object
      title: EvaluationJobConfigCommon1
      description: |
        The evaluation job request configuration.
      required:
        - name
        - model
      properties:
        name:
          type: string
          description: |
            The evaluation job name.
        description:
          type: string
          description: |
            The evaluation job description.
        tags:
          type: array
          items:
            type: string
          description: |
            The evaluation job tags.
        model:
          $ref: '#/components/schemas/ModelRef'
          description: |
            The model to evaluate.
    Ref:
      type: object
      description: Reference by ID
      required:
        - id
      properties:
        id:
          type: string
          description: Unique identifier
    PrimaryScore:
      type: object
      description: Primary score metric configuration
      properties:
        metric:
          type: string
          description: Metric name
        lower_is_better:
          type: boolean
          default: false
          description: Whether lower values indicate better performance
    PassCriteria:
      type: object
      description: Pass/fail threshold criteria
      properties:
        threshold:
          type: number
          format: float
          default: 0
          description: Threshold value
    TestDataRefS3:
      type: object
      description: |
        S3 (or S3-compatible) test data. Only applies when Eval Hub runs the benchmark as a
        **Kubernetes Job** (cluster execution). It does not apply to local-only evaluation runs.

        The test-data **init container** loads the Secret named **`secret_ref`** from the
        job namespace. It expects these keys. **All values must be non-empty** or the init
        container fails:

        - **`AWS_ACCESS_KEY_ID`** — access key
        - **`AWS_SECRET_ACCESS_KEY`** — secret key
        - **`AWS_DEFAULT_REGION`** — region used for signing (for example `us-east-1`)
        - **`AWS_S3_ENDPOINT`** — base URL of the S3 API (AWS regional endpoint or your
          S3-compatible gateway)

        These align with Secrets commonly produced by ODH/RHOAI when an S3 connection is created.
      required:
        - bucket
        - key
        - secret_ref
      properties:
        bucket:
          type: string
          description: S3 bucket name.
        key:
          type: string
          description: |
            Object key or prefix within the bucket. Leading and trailing slashes are
            normalized. The init container lists objects under this prefix and downloads
            each into **`/test_data`**, keeping subdirectory structure relative to the prefix.
        secret_ref:
          type: string
          description: |
            Name of a Kubernetes **`Secret`** in the evaluation job namespace, mounted
            read-only into the test-data init container.
      example:
        bucket: my-eval-bucket
        key: datasets/benchmark-a/v1
        secret_ref: my-s3-connection-secret
    TestDataRef:
      type: object
      description: |
        Reference to external test data for a benchmark.

        When the evaluation runs as a **Kubernetes Job**, setting `s3` makes Eval Hub schedule
        an **init container** on the evaluation job pod before the adapter runs. That init container:

        - Reads **`bucket`**, **`key`**, and **`secret_ref`** from the job spec (same JSON
          mounted into the pod).
        - Uses credentials from the Kubernetes **Secret** named by **`secret_ref`**
          (read-only mount; typical sources include an ODH/RHOAI S3 connection Secret).
        - Lists and downloads all objects under the **`key`** prefix into a shared
          **`emptyDir`** volume at **`/test_data`**, preserving relative paths.
        - Exits; the **adapter container** mounts `/test_data` and reads the files.

        Do not put credentials in the Eval Hub API request body; only reference an existing
        Secret via **`secret_ref`**.
      required:
        - s3
      properties:
        s3:
          $ref: '#/components/schemas/TestDataRefS3'
    BenchmarkConfig:
      type: object
      description: Reference to a benchmark in a collection or evaluation
      allOf:
        - $ref: '#/components/schemas/Ref'
        - type: object
          required:
            - provider_id
          properties:
            provider_id:
              type: string
              description: Evaluation provider ID
            weight:
              type: number
              format: float
              minimum: 0
              maximum: 1
              description: Weight of this benchmark
            primary_score:
              $ref: '#/components/schemas/PrimaryScore'
            pass_criteria:
              $ref: '#/components/schemas/PassCriteria'
            parameters:
              type: object
              additionalProperties: true
              description: Benchmark specific parameters
            test_data_ref:
              $ref: '#/components/schemas/TestDataRef'
              description: |
                Optional external test data (e.g. **S3**). For **Kubernetes Job** runs, may
                trigger a pre-run init container that downloads files to **`/test_data`** for
                the adapter. Omit if the benchmark does not need mounted files.
    ExperimentTag:
      type: object
      description: Tag on an experiment (e.g. MLFlow)
      required:
        - key
        - value
      properties:
        key:
          type: string
          maxLength: 250
          description: Tag key
        value:
          type: string
          maxLength: 5000
          description: Tag value
    ExperimentConfig:
      type: object
      description: Configuration for MLFlow experiment tracking
      properties:
        name:
          type: string
          description: Experiment name
        tags:
          type: array
          items:
            $ref: '#/components/schemas/ExperimentTag'
          maxItems: 20
          description: Experiment tags
        artifact_location:
          type: string
          description: Artifact storage location
    OCICoordinates:
      type: object
      description: OCI artifact coordinates for persistence.
      properties:
        oci_host:
          type: string
          title: OCI Host
          description: OCI registry host (e.g. 'quay.io')
        oci_repository:
          type: string
          title: OCI Repository
          description: OCI repository path (e.g. 'my-org/my-repo')
        oci_tag:
          type: string
          title: OCI Tag
          description: OCI tag (e.g. 'eval-123')
        oci_subject:
          type: string
          title: OCI Subject
          description: Optional OCI subject identifier (in same registry and repo)
        annotations:
          type: object
          additionalProperties:
            type: string
          title: Annotations
          description: Custom annotations for the OCI artifact
      required:
        - oci_host
        - oci_repository
      additionalProperties: false
      title: OCICoordinates
    OCIConnectionConfig:
      type: object
      description: |
        Kubernetes connection configuration for OCI registry authentication. References a Kubernetes Secret of type `kubernetes.io/dockerconfigjson` containing a `.dockerconfigjson` entry with standard Docker registry credentials.
      properties:
        connection:
          type: string
          title: Connection
          description: |
            Name of a Kubernetes Secret (type kubernetes.io/dockerconfigjson) with a ".dockerconfigjson" entry used for OCI registry authentication.
      required:
        - connection
      additionalProperties: false
      title: OCIConnectionConfig
    EvaluationExportsOCI:
      type: object
      description: OCI export configuration for an evaluation job.
      properties:
        coordinates:
          $ref: '#/components/schemas/OCICoordinates'
          description: Coordinates where the OCI artifact should be stored
        k8s:
          $ref: '#/components/schemas/OCIConnectionConfig'
          description: Optional Kubernetes connection for OCI registry authentication
      required:
        - coordinates
      additionalProperties: false
      title: EvaluationExportsOCI
    EvaluationExports:
      type: object
      description: |
        Optional exports configuration for an evaluation job. When provided, the evaluation job results will be exported to the specified location.
      properties:
        oci:
          $ref: '#/components/schemas/EvaluationExportsOCI'
          description: |
            The OCI artifact export configuration.
      additionalProperties: false
      title: EvaluationExports
    EvaluationJobConfigCommon2:
      type: object
      title: EvaluationJobConfigCommon2
      description: |
        The evaluation job request configuration.
      properties:
        pass_criteria:
          $ref: '#/components/schemas/PassCriteria'
          description: |
            The overall pass criteria for the evaluation job.
        experiment:
          $ref: '#/components/schemas/ExperimentConfig'
          description: |
            The MLFlow experiment configuration. When provided, the evaluation job will be tracked in MLFlow.
        exports:
          $ref: '#/components/schemas/EvaluationExports'
          description: |
            Optional exports configuration for the evaluation job. When provided, the evaluation job results will be exported to the specified location.
        custom:
          type: object
          additionalProperties: true
          description: |
            Custom request data. This can be used for user specific job data.
    EvaluationJobConfigBenchmarks:
      allOf:
        - $ref: '#/components/schemas/EvaluationJobConfigCommon1'
        - type: object
          required:
            - benchmarks
          properties:
            benchmarks:
              type: array
              items:
                $ref: '#/components/schemas/BenchmarkConfig'
              description: |
                The evaluation benchmarks to run.
        - $ref: '#/components/schemas/EvaluationJobConfigCommon2'
      title: EvaluationJobConfigBenchmarks
    CollectionRef:
      type: object
      description: |
        The collection to use for the evaluation job.
      required:
        - id
      properties:
        id:
          type: string
          description: |
            The unique identifier of the collection.
        benchmarks:
          type: array
          items:
            $ref: '#/components/schemas/BenchmarkConfig'
          description: |
            The benchmarks supported by this collection. If a benchmark is specified but is not supported by the collection, the evaluation job will fail. This is needed in order to allow overriding the collection benchmarks for a specific evaluation job.
    EvaluationJobConfigCollection:
      allOf:
        - $ref: '#/components/schemas/EvaluationJobConfigCommon1'
        - type: object
          required:
            - collection
          properties:
            collection:
              $ref: '#/components/schemas/CollectionRef'
        - $ref: '#/components/schemas/EvaluationJobConfigCommon2'
      title: EvaluationJobConfigCollection
    EvaluationJobConfig:
      type: object
      title: EvaluationJobConfig
      description: |
        The evaluation job request configuration.
      oneOf:
        - $ref: '#/components/schemas/EvaluationJobConfigBenchmarks'
        - $ref: '#/components/schemas/EvaluationJobConfigCollection'
    EvaluationJobResource:
      type: object
      description: Full evaluation job resource (response)
      allOf:
        - type: object
          properties:
            resource:
              $ref: '#/components/schemas/EvaluationResource'
              description: Resource and evaluation metadata
            status:
              $ref: '#/components/schemas/EvaluationJobStatus'
              description: Current status
            results:
              $ref: '#/components/schemas/EvaluationJobResults'
              description: Results when completed
        - $ref: '#/components/schemas/EvaluationJobConfig'
    EvaluationJobResourceList:
      type: object
      description: List of evaluation job resources with pagination
      allOf:
        - $ref: '#/components/schemas/Page'
        - type: object
          properties:
            items:
              type: array
              items:
                $ref: '#/components/schemas/EvaluationJobResource'
              description: Evaluation job resources
            errors:
              type: array
              items:
                type: string
              description: Non-fatal errors (e.g. partial list)
    EnvVar:
      type: object
      description: Environment variable for job configuration
      properties:
        name:
          type: string
          description: Environment variable name
        value:
          type: string
          description: Environment variable value
      required:
        - name
        - value
    K8sRuntime:
      type: object
      description: Kubernetes runtime configuration for provider evaluation jobs
      properties:
        image:
          type: string
          description: Container image for the evaluation job (e.g. quay.io/evalhub/adapter:latest)
        entrypoint:
          type: array
          items:
            type: string
          description: Container entrypoint command
        cpu_request:
          type: string
          description: CPU request (e.g. 250m)
        memory_request:
          type: string
          description: Memory request (e.g. 512Mi)
        cpu_limit:
          type: string
          description: CPU limit (e.g. 1)
        memory_limit:
          type: string
          description: Memory limit (e.g. 2Gi)
        env:
          type: array
          items:
            $ref: '#/components/schemas/EnvVar'
          description: Environment variables for the job
      required:
        - image
        - entrypoint
    LocalRuntime:
      type: object
      description: Local runtime configuration for running evaluations on the host
      properties:
        command:
          type: string
          description: Command to execute for local evaluation runs
        env:
          type: array
          items:
            $ref: '#/components/schemas/EnvVar'
          description: Environment variables for the local process
      required:
        - command
    Runtime:
      type: object
      description: Provider runtime configuration
      properties:
        k8s:
          $ref: '#/components/schemas/K8sRuntime'
          description: Kubernetes runtime configuration
        local:
          $ref: '#/components/schemas/LocalRuntime'
          description: Local runtime configuration
    BenchmarkResource:
      type: object
      description: Benchmark specification (from a provider)
      properties:
        id:
          type: string
          description: Benchmark ID
        name:
          type: string
          description: Benchmark name
        description:
          type: string
          description: Benchmark description
        category:
          type: string
          description: Category
        metrics:
          type: array
          items:
            type: string
          description: Metric names
        num_few_shot:
          type: integer
          description: Number of few-shot examples
        dataset_size:
          type: integer
          description: Dataset size
        tags:
          type: array
          items:
            type: string
          description: Tags
        primary_score:
          $ref: '#/components/schemas/PrimaryScore'
        pass_criteria:
          $ref: '#/components/schemas/PassCriteria'
    ProviderConfig:
      type: object
      description: Evaluation provider (configuration and available benchmarks)
      properties:
        name:
          type: string
          description: Provider name
        title:
          type: string
          description: Provider display title
        description:
          type: string
          description: Provider description
        tags:
          type: array
          items:
            type: string
          description: Provider tags
        runtime:
          $ref: '#/components/schemas/Runtime'
          description: Provider runtime configuration
        benchmarks:
          type: array
          items:
            $ref: '#/components/schemas/BenchmarkResource'
          description: Benchmarks offered by this provider
      required:
        - name
        - benchmarks
        - runtime
    ProviderResource:
      type: object
      description: Provider resource
      allOf:
        - type: object
          properties:
            resource:
              $ref: '#/components/schemas/Resource'
              description: Resource metadata
        - $ref: '#/components/schemas/ProviderConfig'
    ProviderResourceList:
      type: object
      description: List of provider resources with pagination
      allOf:
        - $ref: '#/components/schemas/Page'
        - type: object
          properties:
            items:
              type: array
              items:
                $ref: '#/components/schemas/ProviderResource'
              description: Provider resources
            errors:
              type: array
              items:
                type: string
              description: Non-fatal errors (e.g. partial list)
    PatchOp:
      type: string
      enum:
        - replace
        - add
        - remove
      description: Patch operation type
    PatchOperation:
      type: object
      description: Single JSON patch operation
      properties:
        op:
          $ref: '#/components/schemas/PatchOp'
        path:
          type: string
          description: JSON Pointer path
        value:
          description: Value for add/replace (omit for remove)
      required:
        - op
        - path
    CollectionConfig:
      type: object
      description: Request to create or update a collection
      properties:
        name:
          type: string
          description: Collection name
        category:
          type: string
          description: Collection category
        description:
          type: string
          description: Optional description
        tags:
          type: array
          items:
            type: string
          description: Tags
        custom:
          type: object
          additionalProperties: true
          description: Custom key-value data
        pass_criteria:
          $ref: '#/components/schemas/PassCriteria'
          description: Pass criteria for the collection
        benchmarks:
          type: array
          items:
            $ref: '#/components/schemas/BenchmarkConfig'
          description: Benchmarks in the collection
      required:
        - name
        - category
        - benchmarks
    CollectionResource:
      type: object
      description: Collection resource
      allOf:
        - type: object
          properties:
            resource:
              $ref: '#/components/schemas/Resource'
              description: Resource metadata
        - $ref: '#/components/schemas/CollectionConfig'
    CollectionResourceList:
      type: object
      description: List of collection resources with pagination
      allOf:
        - $ref: '#/components/schemas/Page'
        - type: object
          properties:
            items:
              type: array
              items:
                $ref: '#/components/schemas/CollectionResource'
              description: Collection resources
  examples:
    BadRequestError:
      summary: Bad request error with status code 400
      value:
        message: The field 'state' is not valid.
        message_code: invalid_value
        trace: b12692e1-8582-4628-88ca-7a13fefb73e2
    AuthenticationError:
      summary: Authentication error with status code 401
      value:
        message: The bearer token is not valid.
        message_code: invalid_auth_token
        trace: b12692e1-8582-4628-88ca-7a13fefb73e2
    AuthorizationError:
      summary: Authorization error with status code 403
      value:
        message: The user does not have access to the endpoint.
        message_code: endpoint_access_forbidden
        trace: b12692e1-8582-4628-88ca-7a13fefb73e2
    NotFoundError:
      summary: Not found error with status code 404
      value:
        message: The resource was not found.
        message_code: not_found
        trace: b12692e1-8582-4628-88ca-7a13fefb73e2
    ConflictError:
      summary: Conflict error with status code 409
      value:
        message: The job 'f02b16a2-1990-4626-b24d-1cff3febdbfb' can not be cancelled because it is 'completed'.
        message_code: conflict
        trace: b12692e1-8582-4628-88ca-7a13fefb73e2
  responses:
    BadRequest:
      description: Bad Request
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Error'
          examples:
            BadRequestError:
              $ref: '#/components/examples/BadRequestError'
    Unauthorized:
      description: Unauthorized
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Error'
          examples:
            AuthenticationError:
              $ref: '#/components/examples/AuthenticationError'
    Forbidden:
      description: Forbidden
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Error'
          examples:
            AuthorizationError:
              $ref: '#/components/examples/AuthorizationError'
    NotFound:
      description: Not Found
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Error'
          examples:
            NotFoundError:
              $ref: '#/components/examples/NotFoundError'
    Conflict:
      description: Conflict
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Error'
          examples:
            ConflictError:
              $ref: '#/components/examples/ConflictError'