feat(specs): add abtests v3

scripts/common.ts

@@ -73,7 +73,7 @@ export const LANGUAGES = [...new Set(Object.values(GENERATORS).map((gen) => gen.
 
 // `crawler` is manually added so we can still bundled and validate the specs
 // the entry can be removed once at least one client is generated
-export const CLIENTS = [...new Set(Object.values(GENERATORS).map((gen) => gen.client)), 'crawler'];
+export const CLIENTS = [...new Set(Object.values(GENERATORS).map((gen) => gen.client)), 'crawler', 'abtesting-v3'];
 
 export async function run(command: string, { errorMessage, cwd, language }: RunOptions = {}): Promise<string> {
   const realCwd = path.resolve(ROOT_DIR, cwd ?? '.');

specs/abtesting-v3/common/parameters.yml

@@ -0,0 +1,158 @@
+# path
+ID:
+  in: path
+  name: id
+  description: Unique A/B test identifier.
+  required: true
+  schema:
+    $ref: '#/abTestID'
+
+# misc
+index:
+  type: string
+  description: Index name of the A/B test variant (case-sensitive).
+  example: 'delcourt_production'
+
+abTestID:
+  type: integer
+  description: Unique A/B test identifier.
+  example: 224
+
+abTestScheduleID:
+  type: integer
+  description: Unique scheduled A/B test identifier.
+  example: 224
+
+endAt:
+  type: string
+  description: End date and time of the A/B test, in RFC 3339 format.
+  example: 2023-06-17T00:00:00Z
+
+createdAt:
+  type: string
+  description: Date and time when the A/B test was created, in RFC 3339 format.
+  example: 2023-06-15T15:06:04.249906Z
+
+updatedAt:
+  type: string
+  description: Date and time when the A/B test was last updated, in RFC 3339 format.
+  example: 2023-06-15T15:06:44.400601Z
+
+scheduledAt:
+  type: string
+  description: Date and time when the A/B test is scheduled to start, in RFC 3339 format.
+  example: 2023-06-15T15:06:44.400601Z
+
+name:
+  type: string
+  description: A/B test name.
+  example: Custom ranking sales rank test
+
+description:
+  type: string
+  description: Description for this variant.
+  example: Current production index
+
+trafficPercentage:
+  type: integer
+  description: Percentage of search requests each variant receives.
+  minimum: 1
+  maximum: 99
+  example: 60
+
+currencies:
+  type: object
+  description: A/B test currencies.
+  example:
+    USD:
+      currency: USD
+      revenue: 120.0
+      mean: 53.7
+      standardDeviation: 12.3
+    EUR:
+      currency: EUR
+      revenue: 100.0
+      mean: 43.7
+      standardDeviation: 10.3
+  additionalProperties:
+    $ref: '#/currency'
+    x-additionalPropertiesName: currency code
+
+currency:
+  type: object
+  properties:
+    currency:
+      type: string
+      description: Currency code.
+      example: 'USD'
+    revenue:
+      type: number
+      format: double
+      description: Revenue for this currency.
+      example: 120.0
+    mean:
+      type: number
+      format: double
+      description: Mean for this currency.
+      example: 53.7
+    standardDeviation:
+      type: number
+      format: double
+      description: Standard deviation for this currency.
+      example: 12.3
+
+filterEffects:
+  type: object
+  description: A/B test filter effects resulting from configuration settings.
+  properties:
+    outliers:
+      title: outliersFilter
+      type: object
+      description: Outliers removed from the A/B test as a result of configuration settings.
+      example:
+        usersCount: 1
+        trackedSearchesCount: 237
+      properties:
+        usersCount:
+          type: integer
+          description: Number of users removed from the A/B test.
+          example: 1
+        trackedSearchesCount:
+          type: integer
+          description: Number of tracked searches removed from the A/B test.
+          example: 237
+    emptySearch:
+      title: emptySearchFilter
+      type: object
+      description: Empty searches removed from the A/B test as a result of configuration settings.
+      example:
+        usersCount: 1
+        trackedSearchesCount: 237
+      properties:
+        usersCount:
+          type: integer
+          description: Number of users removed from the A/B test.
+          example: 1
+        trackedSearchesCount:
+          type: integer
+          description: Number of tracked searches removed from the A/B test.
+          example: 237
+
+metric:
+  type: object
+  description: Defines a metric to be retrieved during an A/B test.
+  properties:
+    name:
+      type: string
+      description: Name of the metric.
+    dimension:
+      type: string
+      description: Dimension of the metric, for example, in case of a revenue metric it could be USD, EUR...
+  required:
+    - name
+  example:
+    - name: revenue
+      dimension: USD
+    - name: conversionRate
+    - name: clickThroughRate
+    - name: trackedSearchCount

specs/abtesting-v3/common/schemas/ABTest.yml

@@ -0,0 +1,137 @@
+ABTests:
+  oneOf:
+    - type: array
+      description: A/B tests.
+      items:
+        $ref: '#/ABTest'
+    - type: 'null'
+
+ABTest:
+  type: object
+  additionalProperties: false
+  properties:
+    abTestID:
+      $ref: '../parameters.yml#/abTestID'
+    updatedAt:
+      $ref: '../parameters.yml#/updatedAt'
+    createdAt:
+      $ref: '../parameters.yml#/createdAt'
+    endAt:
+      $ref: '../parameters.yml#/endAt'
+    name:
+      $ref: '../parameters.yml#/name'
+    status:
+      $ref: '#/Status'
+    variants:
+      $ref: 'Variant.yml#/variants'
+    configuration:
+      $ref: '#/ABTestConfiguration'
+    migratedAbTestID:
+      $ref: '#/MigratedABTestId'
+  required:
+    - status
+    - name
+    - createdAt
+    - endAt
+    - updatedAt
+    - abTestID
+    - variants
+
+Status:
+  type: string
+  description: |
+    A/B test status.
+
+    - `active`. The A/B test is live and search traffic is split between the two variants.
+    - `stopped`. You stopped the A/B test. The A/B test data is still available for analysis.
+    - `expired`. The A/B test was automatically stopped after reaching its end date.
+    - `failed`. Creating the A/B test failed.
+  example: active
+  enum:
+    - active
+    - stopped
+    - expired
+    - failed
+
+ABTestConfiguration:
+  title: configuration
+  type: object
+  description: A/B test configuration.
+  properties:
+    minimumDetectableEffect:
+      $ref: '#/MinimumDetectableEffect'
+    filters:
+      $ref: '#/MetricsFilters'
+    errorCorrection:
+      $ref: '#/ErrorCorrectionType'
+
+MinimumDetectableEffect:
+  type: object
+  description: Configuration for the smallest difference between test variants you want to detect.
+  properties:
+    size:
+      type: number
+      format: double
+      minimum: 0
+      maximum: 1
+      description: |
+        Smallest difference in an observable metric between variants.
+        For example, to detect a 10% difference between variants, set this value to 0.1.
+    metric:
+      $ref: '#/EffectMetric'
+  required:
+    - size
+    - metric
+
+EffectMetric:
+  type: string
+  description: Metric for which you want to detect the smallest relative difference.
+  enum:
+    - addToCartRate
+    - clickThroughRate
+    - conversionRate
+    - purchaseRate
+    - noResultsRate
+
+MigratedABTestId:
+  type: integer
+  description: Unique migrated A/B test identifier.
+  example: 224
+
+MetricsFilter:
+  type: object
+  additionalProperties: false
+  description: |
+    Boolean filter applied to the A/B test population. Each filter targets a boolean metric
+    and decides whether to include (true) or exclude (false) matching records.
+  properties:
+    domain:
+      type: string
+      description: Metric domain (for example `abtesting`, `personalization`).
+      example: abtesting
+    name:
+      type: string
+      description: Public metric name.
+      example: isOutlier
+    trackEffects:
+      type: boolean
+      description: Whether the experiment should record the effects of this filter.
+    includes:
+      type: boolean
+      description: If true, keep items that match the filter; if false, exclude them.
+  required:
+    - domain
+    - name
+
+MetricsFilters:
+  type: array
+  description: List of metric filters applied to the test population.
+  items:
+    $ref: '#/MetricsFilter'
+
+ErrorCorrectionType:
+  type: string
+  description: Multiple-testing correction method applied when evaluating metric significance.
+  enum:
+    - bonferroni
+    - benjamini-hochberg

specs/abtesting-v3/common/schemas/ABTestResponse.yml

@@ -0,0 +1,14 @@
+ABTestResponse:
+  type: object
+  additionalProperties: false
+  properties:
+    index:
+      $ref: '../parameters.yml#/index'
+    abTestID:
+      $ref: '../parameters.yml#/abTestID'
+    taskID:
+      $ref: '../../../common/responses/common.yml#/taskID'
+  required:
+    - abTestID
+    - index
+    - taskID

specs/abtesting-v3/common/schemas/AddABTestsVariant.yml

@@ -0,0 +1,38 @@
+AddABTestsVariant:
+  oneOf:
+    - $ref: '#/abTestsVariant'
+    - $ref: '#/abTestsVariantSearchParams'
+
+abTestsVariantSearchParams:
+  allOf:
+    - $ref: '#/abTestsVariant'
+    - $ref: '#/customSearchParams'
+
+abTestsVariant:
+  type: object
+  additionalProperties: false
+  properties:
+    index:
+      $ref: '../parameters.yml#/index'
+    trafficPercentage:
+      $ref: '../parameters.yml#/trafficPercentage'
+    description:
+      $ref: '../parameters.yml#/description'
+  required:
+    - index
+    - trafficPercentage
+
+customSearchParams:
+  type: object
+  description: |
+    Search parameters to add to the test variant.
+    Only use this parameter if the two variants use the same index.
+  example: {'typoTolerance': false, 'synonyms': false}
+  additionalProperties: false
+  properties:
+    customSearchParameters:
+      type: object
+  required:
+    - customSearchParameters
+  x-discriminator-fields:
+    - customSearchParameters

specs/abtesting-v3/common/schemas/EstimateABTestResponse.yml

@@ -0,0 +1,18 @@
+EstimateABTestResponse:
+  type: object
+  properties:
+    durationDays:
+      type: integer
+      format: int64
+      description: Estimated number of days needed to reach the sample sizes required for detecting the configured effect. This value is based on historical traffic.
+      example: 21
+    sampleSizes:
+      type: array
+      description: |
+        Sample size estimates for each variant. The first element is the control variant.
+        Each element is the estimated number of searches required to achieve the desired statistical significance.
+      items:
+        type: integer
+        format: int64
+        description: Number of tracked searches needed to be able to detect the configured effect for the control variant.
+        example: 23415

specs/abtesting-v3/common/schemas/ScheduleABTestResponse.yml

@@ -0,0 +1,8 @@
+ScheduleABTestResponse:
+  type: object
+  additionalProperties: false
+  properties:
+    abTestScheduleID:
+      $ref: '../parameters.yml#/abTestScheduleID'
+  required:
+    - abTestScheduleID