feat(javascript): add replaceAllObjectsWithTransformation

playground/javascript/node/algoliasearch.ts

@@ -136,18 +136,24 @@ async function testAlgoliasearchBridgeIngestion() {
   // Init client with appId and apiKey
   const client = algoliasearch(appId, adminApiKey, { transformation: { region: 'eu' } });
 
-  await client.saveObjectsWithTransformation({
-    indexName: 'foo',
-    objects: [{ objectID: 'foo', data: { baz: 'baz', win: 42 } }],
-    waitForTasks: true,
-  });
-
-  await client.partialUpdateObjectsWithTransformation({
-    indexName: 'foo',
-    objects: [{ objectID: 'foo', data: { baz: 'baz', win: 42 } }],
-    waitForTasks: true,
-    createIfNotExists: false,
-  });
+  // console.log('saveObjectsWithTransformation', await client.saveObjectsWithTransformation({
+  //   indexName: 'foo',
+  //   objects: [{ objectID: 'foo', data: { baz: 'baz', win: 42 } }],
+  //   waitForTasks: true,
+  // }));
+  //
+  // console.log('partialUpdateObjectsWithTransformation', await client.partialUpdateObjectsWithTransformation({
+  //   indexName: 'foo',
+  //   objects: [{ objectID: 'foo', data: { baz: 'baz', win: 42 } }],
+  //   waitForTasks: true,
+  //   createIfNotExists: false,
+  // }));
+
+  console.log('replaceAllObjectsWithTransformation', await client.replaceAllObjectsWithTransformation({
+    indexName: 'boyd',
+    objects: [{ objectID: 'foo', data: { baz: 'baz', win: 42 } }, { objectID: 'bar', data: { baz: 'baz', win: 24 } }],
+    batchSize: 2
+  }));
 }
 
 // testAlgoliasearch();

scripts/cts/runCts.ts

@@ -13,6 +13,7 @@ import { assertPushMockValid } from './testServer/pushMock.ts';
 import { assertValidReplaceAllObjects } from './testServer/replaceAllObjects.ts';
 import { assertValidReplaceAllObjectsFailed } from './testServer/replaceAllObjectsFailed.ts';
 import { assertValidReplaceAllObjectsScopes } from './testServer/replaceAllObjectsScopes.ts';
+import { assertValidReplaceAllObjectsWithTransformation } from './testServer/replaceAllObjectsWithTransformation.ts';
 import { assertValidTimeouts } from './testServer/timeout.ts';
 import { assertValidWaitForApiKey } from './testServer/waitFor.ts';
 
@@ -154,6 +155,7 @@ export async function runCts(
     assertValidTimeouts(languages.length);
     assertChunkWrapperValid(languages.length - skip('dart'));
     assertValidReplaceAllObjects(languages.length - skip('dart'));
+    assertValidReplaceAllObjectsWithTransformation(only('javascript'));
     assertValidAccountCopyIndex(only('javascript'));
     assertValidReplaceAllObjectsFailed(languages.length - skip('dart'));
     assertValidReplaceAllObjectsScopes(languages.length - skip('dart'));

scripts/cts/testServer/index.ts

@@ -17,6 +17,7 @@ import { pushMockServer } from './pushMock.ts';
 import { replaceAllObjectsServer } from './replaceAllObjects.ts';
 import { replaceAllObjectsServerFailed } from './replaceAllObjectsFailed.ts';
 import { replaceAllObjectsScopesServer } from './replaceAllObjectsScopes.ts';
+import { replaceAllObjectsWithTransformationServer } from './replaceAllObjectsWithTransformation.ts';
 import { timeoutServer } from './timeout.ts';
 import { timeoutServerBis } from './timeoutBis.ts';
 import { waitForApiKeyServer } from './waitFor.ts';
@@ -37,6 +38,7 @@ export async function startTestServer(suites: Record<CTSType, boolean>): Promise
       apiKeyServer(),
       algoliaMockServer(),
       pushMockServer(),
+      replaceAllObjectsWithTransformationServer(),
     );
   }
   if (suites.benchmark) {

scripts/cts/testServer/replaceAllObjectsWithTransformation.ts

@@ -0,0 +1,132 @@
+import type { Server } from 'http';
+
+import { expect } from 'chai';
+import type { Express } from 'express';
+import express from 'express';
+
+import { setupServer } from './index.ts';
+
+const raowtState: Record<
+  string,
+  {
+    copyCount: number;
+    pushCount: number;
+    tmpIndexName: string;
+    waitTaskCount: number;
+    waitingForFinalWaitTask: boolean;
+    successful: boolean;
+  }
+> = {};
+
+export function assertValidReplaceAllObjectsWithTransformation(expectedCount: number): void {
+  expect(Object.keys(raowtState)).to.have.length(expectedCount);
+  for (const lang in raowtState) {
+    expect(raowtState[lang].successful).to.equal(true);
+  }
+}
+
+function addRoutes(app: Express): void {
+  app.use(express.urlencoded({ extended: true }));
+  app.use(
+    express.json({
+      type: ['application/json', 'text/plain'], // the js client sends the body as text/plain
+    }),
+  );
+
+  app.post('/1/indexes/:indexName/operation', (req, res) => {
+    expect(req.params.indexName).to.match(/^cts_e2e_replace_all_objects_with_transformation_(.*)$/);
+
+    switch (req.body.operation) {
+      case 'copy': {
+        expect(req.params.indexName).to.not.include('tmp');
+        expect(req.body.destination).to.include('tmp');
+        expect(req.body.scope).to.deep.equal(['settings', 'rules', 'synonyms']);
+
+        const lang = req.params.indexName.replace('cts_e2e_replace_all_objects_with_transformation_', '');
+        if (!raowtState[lang] || raowtState[lang].successful) {
+          raowtState[lang] = {
+            copyCount: 1,
+            pushCount: 0,
+            waitTaskCount: 0,
+            tmpIndexName: req.body.destination,
+            waitingForFinalWaitTask: false,
+            successful: false,
+          };
+        } else {
+          raowtState[lang].copyCount++;
+        }
+
+        res.json({ taskID: 123 + raowtState[lang].copyCount, updatedAt: '2021-01-01T00:00:00.000Z' });
+        break;
+      }
+      case 'move': {
+        const lang = req.body.destination.replace('cts_e2e_replace_all_objects_with_transformation_', '');
+        expect(raowtState).to.include.keys(lang);
+        expect(raowtState[lang]).to.deep.equal({
+          copyCount: 2,
+          pushCount: 10,
+          waitTaskCount: 2,
+          tmpIndexName: req.params.indexName,
+          waitingForFinalWaitTask: false,
+          successful: false,
+        });
+
+        expect(req.body.scope).to.equal(undefined);
+
+        raowtState[lang].waitingForFinalWaitTask = true;
+
+        res.json({ taskID: 777, updatedAt: '2021-01-01T00:00:00.000Z' });
+
+        break;
+      }
+      default:
+        res.status(400).json({
+          message: `invalid operation: ${req.body.operation}, body: ${JSON.stringify(req.body)}`,
+        });
+    }
+  });
+
+  app.post('/1/push/:indexName', (req, res) => {
+    const lang = req.params.indexName.match(
+      /^cts_e2e_replace_all_objects_with_transformation_(.*)_tmp_\d+$/,
+    )?.[1] as string;
+    expect(raowtState).to.include.keys(lang);
+    expect(req.body.action === 'addObject').to.equal(true);
+
+    raowtState[lang].pushCount += req.body.records.length;
+
+    res.json({
+      runID: 'b1b7a982-524c-40d2-bb7f-48aab075abda',
+      eventID: `113b2068-6337-4c85-b5c2-e7b213d8292${raowtState[lang].pushCount}`,
+      message: 'OK',
+      createdAt: '2022-05-12T06:24:30.049Z',
+    });
+  });
+
+  app.get('/1/runs/:runID/events/:eventID', (req, res) => {
+    res.json({ status: 'finished' });
+  });
+
+  app.get('/1/indexes/:indexName/task/:taskID', (req, res) => {
+    const lang = req.params.indexName.match(
+      /^cts_e2e_replace_all_objects_with_transformation_(.*)_tmp_\d+$/,
+    )?.[1] as string;
+    expect(raowtState).to.include.keys(lang);
+
+    raowtState[lang].waitTaskCount++;
+    if (raowtState[lang].waitingForFinalWaitTask) {
+      expect(req.params.taskID).to.equal('777');
+      expect(raowtState[lang].waitTaskCount).to.equal(3);
+
+      raowtState[lang].successful = true;
+    }
+
+    res.json({ status: 'published', updatedAt: '2021-01-01T00:00:00.000Z' });
+  });
+}
+
+export function replaceAllObjectsWithTransformationServer(): Promise<Server> {
+  // this server is used to simulate the responses for the replaceAllObjectsWithTransformationServer method,
+  // and uses a state machine to determine if the logic is correct.
+  return setupServer('replaceAllObjectsWithTransformationServer', 6690, addRoutes);
+}

specs/search/helpers/chunkedPush.yml

@@ -0,0 +1,55 @@
+method:
+  get:
+    x-helper: true
+    tags:
+      - Records
+    x-available-languages:
+      - javascript
+    operationId: chunkedPush
+    summary: Replace all records in an index
+    description: |
+      Helper: Chunks the given `objects` list in subset of 1000 elements max in order to make it fit in `push` requests by leveraging the Transformation pipeline setup in the Push connector (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/).
+    parameters:
+      - in: query
+        name: indexName
+        description: The `indexName` to replace `objects` in.
+        required: true
+        schema:
+          type: string
+      - in: query
+        name: objects
+        description: List of objects to replace the current objects with.
+        required: true
+        schema:
+          type: array
+          items:
+            type: object
+      - in: query
+        name: action
+        description: The `batch` `action` to perform on the given array of `objects`, defaults to `addObject`.
+        required: false
+        schema:
+          $ref: '../../common/schemas/Batch.yml#/action'
+      - in: query
+        name: waitForTasks
+        description: Whether or not we should wait until every `batch` tasks has been processed, this operation may slow the total execution time of this method but is more reliable.
+        required: false
+        schema:
+          type: boolean
+      - in: query
+        name: batchSize
+        description: The size of the chunk of `objects`. The number of `batch` calls will be equal to `length(objects) / batchSize`. Defaults to 1000.
+        required: false
+        schema:
+          type: integer
+    responses:
+      '200':
+        description: OK
+        content:
+          application/json:
+            schema:
+              type: array
+              items:
+                $ref: '../../common/schemas/ingestion/WatchResponse.yml'
+      '400':
+        $ref: '../../common/responses/IndexNotFound.yml'

specs/search/helpers/replaceAllObjectsWithTransformation.yml

@@ -0,0 +1,82 @@
+method:
+  get:
+    x-helper: true
+    tags:
+      - Records
+    x-available-languages:
+      - javascript
+    operationId: replaceAllObjectsWithTransformation
+    summary: Replace all records in an index
+    description: |
+      Replace all records from your index with a new set of records by leveraging the Transformation pipeline setup in the Push connector (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/).
+
+      This method lets you replace all records in your index without downtime. It performs these operations:
+        1. Copy settings, synonyms, and rules from your original index to a temporary index.
+        2. Add your new records to the temporary index.
+        3. Replace your original index with the temporary index.
+
+      Use the safe parameter to ensure that these (asynchronous) operations are performed in sequence.
+      If there's an error duing one of these steps, the temporary index won't be deleted.
+      This operation is rate-limited.
+      This method creates a temporary index: your record count is temporarily doubled. Algolia doesn't count the three days with the highest number of records towards your monthly usage.
+      If you're on a legacy plan (before July 2020), this method counts two operations towards your usage (in addition to the number of records): copySettings and moveIndex.
+      The API key you use for this operation must have access to the index YourIndex and the temporary index YourIndex_tmp.
+    parameters:
+      - in: query
+        name: indexName
+        description: The `indexName` to replace `objects` in.
+        required: true
+        schema:
+          type: string
+      - in: query
+        name: objects
+        description: List of objects to replace the current objects with.
+        required: true
+        schema:
+          type: array
+          items:
+            type: object
+      - in: query
+        name: batchSize
+        description: The size of the chunk of `objects`. The number of `batch` calls will be equal to `length(objects) / batchSize`. Defaults to 1000.
+        required: false
+        schema:
+          type: integer
+          default: 1000
+      - in: query
+        name: scopes
+        description: List of scopes to kepp in the index. Defaults to `settings`, `synonyms`, and `rules`.
+        required: false
+        schema:
+          type: array
+          items:
+            $ref: '../common/enums.yml#/scopeType'
+    responses:
+      '200':
+        description: OK
+        content:
+          application/json:
+            schema:
+              $ref: '#/replaceAllObjectsWithTransformationResponse'
+      '400':
+        $ref: '../../common/responses/IndexNotFound.yml'
+
+replaceAllObjectsWithTransformationResponse:
+  type: object
+  additionalProperties: false
+  properties:
+    copyOperationResponse:
+      description: The response of the `operationIndex` request for the `copy` operation.
+      $ref: '../../common/responses/common.yml#/updatedAtResponse'
+    watchResponses:
+      type: array
+      description: The response of the `push` request(s).
+      items:
+        $ref: '../../common/schemas/ingestion/WatchResponse.yml'
+    moveOperationResponse:
+      description: The response of the `operationIndex` request for the `move` operation.
+      $ref: '../../common/responses/common.yml#/updatedAtResponse'
+  required:
+    - copyOperationResponse
+    - watchResponses
+    - moveOperationResponse

specs/search/spec.yml

@@ -378,6 +378,9 @@ paths:
   /replaceAllObjects:
     $ref: 'helpers/replaceAllObjects.yml#/method'
 
+  /replaceAllObjectsWithTransformation:
+    $ref: 'helpers/replaceAllObjectsWithTransformation.yml#/method'
+
   /chunkedBatch:
     $ref: 'helpers/chunkedBatch.yml#/method'