Make debug logging of typesense imports expicit and configurable

.eslintrc.js

@@ -6,7 +6,7 @@ module.exports = {
     jest: true,
   },
   parserOptions: {
-    ecmaVersion: 2020,
+    ecmaVersion: 2022,
   },
   extends: ["eslint:recommended", "google"],
   rules: {

extension.yaml

@@ -108,6 +108,18 @@ params:
         value: true
     default: false
     required: false
+  - param: LOG_TYPESENSE_INSERTS
+    label: Log Typesense Inserts for Debugging
+    description: >-
+      Should data inserted into Typesense be logged in Cloud Logging? This can be useful for debugging, but should not be enabled in production.
+    type: select
+    options:
+      - label: No
+        value: false
+      - label: Yes
+        value: true
+    default: false
+    required: false
   - param: LOCATION
     label: Cloud Functions location
     description: >-

functions/src/backfill.js

@@ -68,7 +68,11 @@ module.exports = functions.firestore.document(config.typesenseBackfillTriggerDoc
             const pathParams = utils.pathMatchesSelector(docPath, config.firestoreCollectionPath);
 
             if (!isGroupQuery || (isGroupQuery && pathParams !== null)) {
-              return await utils.typesenseDocumentFromSnapshot(doc, pathParams);
+              const typesenseDocument = await utils.typesenseDocumentFromSnapshot(doc, pathParams);
+              if (config.shouldLogTypesenseInserts) {
+                functions.logger.debug(`Backfilling document ${JSON.stringify(typesenseDocument)}`);
+              }
+              return typesenseDocument;
             } else {
               return null;
             }

functions/src/config.js

@@ -6,6 +6,7 @@ module.exports = {
         .map((f) => f.trim())
         .filter((f) => f),
   shouldFlattenNestedDocuments: process.env.FLATTEN_NESTED_DOCUMENTS === "true",
+  shouldLogTypesenseInserts: process.env.LOG_TYPESENSE_INSERTS === "true",
   typesenseHosts:
     (process.env.TYPESENSE_HOSTS || "").split(",").map((e) => e.trim()),
   typesensePort: process.env.TYPESENSE_PORT || 443,

functions/src/indexOnWrite.js

@@ -22,7 +22,11 @@ module.exports = functions.firestore.document(config.firestoreCollectionPath)
         const latestSnapshot = await snapshot.after.ref.get();
         const typesenseDocument = await utils.typesenseDocumentFromSnapshot(latestSnapshot, context.params);
 
-        functions.logger.debug(`Upserting document ${JSON.stringify(typesenseDocument)}`);
+        if (config.shouldLogTypesenseInserts) {
+          functions.logger.debug(`Upserting document ${JSON.stringify(typesenseDocument)}`);
+        } else {
+          functions.logger.debug(`Upserting document ${typesenseDocument.id}`);
+        }
         return await typesense
             .collections(encodeURIComponent(config.typesenseCollectionName))
             .documents()

functions/src/utils.js

@@ -145,8 +145,6 @@ exports.typesenseDocumentFromSnapshot = async (firestoreDocumentSnapshot, contex
   // using flat to flatten nested objects for older versions of Typesense that did not support nested fields
   // https://typesense.org/docs/0.22.2/api/collections.html#indexing-nested-fields
   const typesenseDocument = config.shouldFlattenNestedDocuments ? flattenDocument(mappedDocument) : mappedDocument;
-  console.log("typesenseDocument", typesenseDocument);
-
   typesenseDocument.id = firestoreDocumentSnapshot.id;
 
   if (contextParams && Object.entries(contextParams).length) {

package.json

@@ -5,9 +5,9 @@
     "emulator": "cross-env DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:start --import=emulator_data",
     "export": "firebase emulators:export emulator_data",
     "test": "npm run test-part-1 && npm run test-part-2 && npm run test-part-3",
-    "test-part-1": "cp -f extensions/test-params-flatten-nested-true.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-true.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testPathIgnorePatterns=\"WithoutFlattening\" --testPathIgnorePatterns=\"Subcollection\"'",
+    "test-part-1": "cp -f extensions/test-params-flatten-nested-true.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-true.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"WithFlattening\" --testRegex=\"backfill.spec\"'",
     "test-part-2": "cp -f extensions/test-params-flatten-nested-false.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"WithoutFlattening\"'",
-    "test-part-3": "cp -f extensions/test-params-subcategory-flatten-nested-false.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-subcategory-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"Subcollection\"'",
+    "test-part-3": "jest --testRegex=\"writeLogging\" --testRegex=\"Subcollection\" --detectOpenHandles",
     "typesenseServer": "docker compose up",
     "lint:fix": "eslint . --fix",
     "lint": "eslint ."

test/backfillSubcollection.spec.js

@@ -1,59 +1,56 @@
-const firebase = require("firebase-admin");
-const config = require("../functions/src/config.js");
-const typesense = require("../functions/src/createTypesenseClient.js")();
-
-const app = firebase.initializeApp({
-  // Use a special URL to talk to the Realtime Database emulator
-  databaseURL: `${process.env.FIREBASE_DATABASE_EMULATOR_HOST}?ns=${process.env.GCLOUD_PROJECT}`,
-  projectId: process.env.GCLOUD_PROJECT,
-});
-const firestore = app.firestore();
+const {TestEnvironment} = require("./support/testEnvironment");
+
+// test case configs
+const TEST_FIRESTORE_PARENT_COLLECTION_PATH = "users";
+const TEST_FIRESTORE_CHILD_FIELD_NAME = "books";
 
 describe("backfillSubcollection", () => {
-  const parentCollectionPath = process.env.TEST_FIRESTORE_PARENT_COLLECTION_PATH;
+  let testEnvironment;
+
+  const parentCollectionPath = TEST_FIRESTORE_PARENT_COLLECTION_PATH;
   const unrelatedCollectionPath = "unrelatedCollectionToNotBackfill";
-  const childFieldName = process.env.TEST_FIRESTORE_CHILD_FIELD_NAME;
+  const childFieldName = TEST_FIRESTORE_CHILD_FIELD_NAME;
   let parentIdField = null;
 
-  beforeAll(() => {
-    const matches = config.firestoreCollectionPath.match(/{([^}]+)}/g);
-    expect(matches).toBeDefined();
-    expect(matches.length).toBe(1);
+  let config = null;
+  let firestore = null;
+  let typesense = null;
+
+  beforeAll((done) => {
+    testEnvironment = new TestEnvironment({
+      dotenvPath: "extensions/test-params-subcategory-flatten-nested-false.local.env",
+      outputAllEmulatorLogs: true,
+    });
+    testEnvironment.setupTestEnvironment((err) => {
+      const matches = testEnvironment.config.firestoreCollectionPath.match(/{([^}]+)}/g);
+      expect(matches).toBeDefined();
+      expect(matches.length).toBe(1);
+
+      parentIdField = matches[0].replace(/{|}/g, "");
+      expect(parentIdField).toBeDefined();
+
+      config = testEnvironment.config;
+      firestore = testEnvironment.firestore;
+      typesense = testEnvironment.typesense;
+      done();
+    });
+  });
 
-    parentIdField = matches[0].replace(/{|}/g, "");
-    expect(parentIdField).toBeDefined();
+  afterAll(async () => {
+    await testEnvironment.teardownTestEnvironment();
   });
 
   beforeEach(async () => {
-    // Clear the database between tests
+    // For subcollections, need special handling to clear parent collection. Deleting here triggers firebase functions
     await firestore.recursiveDelete(firestore.collection(parentCollectionPath));
     await firestore.recursiveDelete(firestore.collection(unrelatedCollectionPath));
 
-    // Clear any previously created collections
-    try {
-      await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete();
-    } catch (e) {
-      console.info(`${config.typesenseCollectionName} collection not found, proceeding...`);
-    }
-
-    // Create a new Typesense collection
-    return typesense.collections().create({
-      name: config.typesenseCollectionName,
-      fields: [
-        {name: ".*", type: "auto"},
-      ],
-    });
-  });
-
-  afterAll(async () => {
-    // clean up the firebase app after all tests have run
-    await app.delete();
+    await testEnvironment.clearAllData();
   });
 
   describe("when firestore_collections is not specified", () => {
     it("backfills existing Firestore data in all collections to Typesense" +
-      " when `trigger: true` is set " +
-      ` in ${config.typesenseBackfillTriggerDocumentInFirestore}`, async () => {
+      " when `trigger: true` is set on trigger document", async () => {
       const parentDocData = {
         nested_field: {
           field1: "value1",
@@ -67,7 +64,7 @@ describe("backfillSubcollection", () => {
       };
 
       // create parent document in Firestore
-      const parentDocRef = await firestore.collection(parentCollectionPath).add(parentDocData);
+      const parentDocRef = await testEnvironment.firestore.collection(parentCollectionPath).add(parentDocData);
 
       // create a subcollection with document under the parent document
       const subCollectionRef = parentDocRef.collection(childFieldName);
@@ -78,24 +75,24 @@ describe("backfillSubcollection", () => {
 
       // The above will automatically add the document to Typesense,
       // so delete it so we can test backfill
-      await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete();
-      await typesense.collections().create({
-        name: config.typesenseCollectionName,
+      await testEnvironment.typesense.collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName)).delete();
+      await testEnvironment.typesense.collections().create({
+        name: testEnvironment.config.typesenseCollectionName,
         fields: [
           {name: ".*", type: "auto"},
         ],
       });
 
-      await firestore
-        .collection(config.typesenseBackfillTriggerDocumentInFirestore.split("/")[0])
+      await testEnvironment.firestore
+        .collection(testEnvironment.config.typesenseBackfillTriggerDocumentInFirestore.split("/")[0])
         .doc("backfill")
         .set({trigger: true});
       // Wait for firestore cloud function to write to Typesense
       await new Promise((r) => setTimeout(r, 2000));
 
       // Check that the data was backfilled
-      const typesenseDocsStr = await typesense
-        .collections(encodeURIComponent(config.typesenseCollectionName))
+      const typesenseDocsStr = await testEnvironment.typesense
+        .collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName))
         .documents()
         .export();
       const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s));
@@ -113,8 +110,7 @@ describe("backfillSubcollection", () => {
   describe("when firestore_collections is specified", () => {
     describe("when firestore_collections includes this collection", () => {
       it("backfills existing Firestore data in this particular collection to Typesense" +
-        " when `trigger: true` is set " +
-        ` in ${config.typesenseBackfillTriggerDocumentInFirestore}`, async () => {
+        " when `trigger: true` is set on trigger document", async () => {
         const parentDocData = {
           nested_field: {
             field1: "value1",
@@ -163,6 +159,7 @@ describe("backfillSubcollection", () => {
           .documents()
           .export();
         const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s));
+        console.log(typesenseDocs);
         expect(typesenseDocs.length).toBe(1);
         expect(typesenseDocs[0]).toStrictEqual({
           id: subDocRef.id,
@@ -175,8 +172,7 @@ describe("backfillSubcollection", () => {
 
     describe("when firestore_collections does not include this collection", () => {
       it("does not backfill existing Firestore data in this particular collection to Typesense" +
-        " when `trigger: true` is set " +
-        ` in ${config.typesenseBackfillTriggerDocumentInFirestore}`, async () => {
+        " when `trigger: true` is set on trigger document", async () => {
         const parentDocData = {
           nested_field: {
             field1: "value1",
@@ -194,7 +190,7 @@ describe("backfillSubcollection", () => {
 
         // create a subcollection with document under the parent document
         const subCollectionRef = parentDocRef.collection(childFieldName);
-        await subCollectionRef.add(subDocData);
+        const subDocRef = await subCollectionRef.add(subDocData);
         // Wait for firestore cloud function to write to Typesense
         await new Promise((r) => setTimeout(r, 2000));
 
@@ -224,6 +220,15 @@ describe("backfillSubcollection", () => {
           .documents()
           .export();
         expect(typesenseDocsStr).toEqual("");
+
+        // Check that the error was logged
+        testEnvironment.resetCapturedEmulatorLogs();
+        subDocRef.delete();
+        await new Promise((r) => setTimeout(r, 5000));
+
+        expect(testEnvironment.capturedEmulatorLogs).toContain(
+          `Could not find a document with id: ${subDocRef.id}`,
+        );
       });
     });
   });

test/indexOnWriteSubcollection.spec.js

@@ -1,50 +1,50 @@
-const firebase = require("firebase-admin");
-const config = require("../functions/src/config.js");
-const typesense = require("../functions/src/createTypesenseClient.js")();
-
-const app = firebase.initializeApp({
-  // Use a special URL to talk to the Realtime Database emulator
-  databaseURL: `${process.env.FIREBASE_DATABASE_EMULATOR_HOST}?ns=${process.env.GCLOUD_PROJECT}`,
-  projectId: process.env.GCLOUD_PROJECT,
-});
-const firestore = app.firestore();
+const {TestEnvironment} = require("./support/testEnvironment");
+
+// test case configs
+const TEST_FIRESTORE_PARENT_COLLECTION_PATH = "users";
+const TEST_FIRESTORE_CHILD_FIELD_NAME = "books";
+
 
 describe("indexOnWriteSubcollection", () => {
-  const parentCollectionPath = process.env.TEST_FIRESTORE_PARENT_COLLECTION_PATH;
-  const childFieldName = process.env.TEST_FIRESTORE_CHILD_FIELD_NAME;
+  let testEnvironment;
+
+  const parentCollectionPath = TEST_FIRESTORE_PARENT_COLLECTION_PATH;
+  const childFieldName = TEST_FIRESTORE_CHILD_FIELD_NAME;
   let parentIdField = null;
 
-  beforeAll(async () => {
-    const matches = config.firestoreCollectionPath.match(/{([^}]+)}/g);
-    expect(matches).toBeDefined();
-    expect(matches.length).toBe(1);
+  let config = null;
+  let firestore = null;
+  let typesense = null;
 
-    parentIdField = matches[0].replace(/{|}/g, "");
-    expect(parentIdField).toBeDefined();
+  beforeAll((done) => {
+    testEnvironment = new TestEnvironment({
+      dotenvPath: "extensions/test-params-subcategory-flatten-nested-false.local.env",
+      outputAllEmulatorLogs: true,
+    });
+    testEnvironment.setupTestEnvironment((err) => {
+      const matches = testEnvironment.config.firestoreCollectionPath.match(/{([^}]+)}/g);
+      expect(matches).toBeDefined();
+      expect(matches.length).toBe(1);
+
+      parentIdField = matches[0].replace(/{|}/g, "");
+      expect(parentIdField).toBeDefined();
+
+      config = testEnvironment.config;
+      firestore = testEnvironment.firestore;
+      typesense = testEnvironment.typesense;
+      done();
+    });
+  });
+
+  afterAll(async () => {
+    await testEnvironment.teardownTestEnvironment();
   });
 
   beforeEach(async () => {
-    // delete the Firestore collection
+    // For subcollections, need special handling to clear parent collection. Deleting here triggers firebase functions
     await firestore.recursiveDelete(firestore.collection(parentCollectionPath));
 
-    // Clear any previously created collections
-    try {
-      await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete();
-    } catch (e) {
-      console.info(`${config.typesenseCollectionName} collection not found, proceeding...`);
-    }
-
-    // recreate the Typesense collection
-    await typesense.collections().create({
-      name: config.typesenseCollectionName,
-      fields: [{name: ".*", type: "auto"}],
-      enable_nested_fields: true,
-    });
-  });
-
-  afterAll(async () => {
-    // clean up the whole firebase app
-    await app.delete();
+    await testEnvironment.clearAllData();
   });
 
   describe("Backfill from dynamic subcollections", () => {