From 55fcb8cc6c198fabc6b5b1dd352c3dbfafdb1829 Mon Sep 17 00:00:00 2001 From: James Sasitorn Date: Sun, 29 Dec 2024 16:23:09 -0800 Subject: [PATCH 1/3] Add explicit config support for debug logging of typesense imports. This standardizes logging across both .onWrite and .backfill, and allows disabling logging for production builds and improved data privacy. Option can be turned off an off in firebase extension config using LOG_TYPESENSE_INSERTS --- extension.yaml | 12 ++++++++++++ functions/src/backfill.js | 6 +++++- functions/src/config.js | 1 + functions/src/indexOnWrite.js | 6 +++++- functions/src/utils.js | 2 -- 5 files changed, 23 insertions(+), 4 deletions(-) diff --git a/extension.yaml b/extension.yaml index ab889a8..8f37063 100644 --- a/extension.yaml +++ b/extension.yaml @@ -108,6 +108,18 @@ params: value: true default: false required: false + - param: LOG_TYPESENSE_INSERTS + label: Log Typesense Inserts for Debugging + description: >- + Should data inserted into Typesense be logged in Cloud Logging? This can be useful for debugging, but should not be enabled in production. + type: select + options: + - label: No + value: false + - label: Yes + value: true + default: false + required: false - param: LOCATION label: Cloud Functions location description: >- diff --git a/functions/src/backfill.js b/functions/src/backfill.js index a3641be..38ac5bb 100644 --- a/functions/src/backfill.js +++ b/functions/src/backfill.js @@ -68,7 +68,11 @@ module.exports = functions.firestore.document(config.typesenseBackfillTriggerDoc const pathParams = utils.pathMatchesSelector(docPath, config.firestoreCollectionPath); if (!isGroupQuery || (isGroupQuery && pathParams !== null)) { - return await utils.typesenseDocumentFromSnapshot(doc, pathParams); + const typesenseDocument = await utils.typesenseDocumentFromSnapshot(doc, pathParams); + if (config.shouldLogTypesenseInserts) { + functions.logger.debug(`Backfilling document ${JSON.stringify(typesenseDocument)}`); + } + return typesenseDocument; } else { return null; } diff --git a/functions/src/config.js b/functions/src/config.js index efa84ed..502fb3a 100644 --- a/functions/src/config.js +++ b/functions/src/config.js @@ -6,6 +6,7 @@ module.exports = { .map((f) => f.trim()) .filter((f) => f), shouldFlattenNestedDocuments: process.env.FLATTEN_NESTED_DOCUMENTS === "true", + shouldLogTypesenseInserts: process.env.LOG_TYPESENSE_INSERTS === "true", typesenseHosts: (process.env.TYPESENSE_HOSTS || "").split(",").map((e) => e.trim()), typesensePort: process.env.TYPESENSE_PORT || 443, diff --git a/functions/src/indexOnWrite.js b/functions/src/indexOnWrite.js index 2db5935..eef0f51 100644 --- a/functions/src/indexOnWrite.js +++ b/functions/src/indexOnWrite.js @@ -22,7 +22,11 @@ module.exports = functions.firestore.document(config.firestoreCollectionPath) const latestSnapshot = await snapshot.after.ref.get(); const typesenseDocument = await utils.typesenseDocumentFromSnapshot(latestSnapshot, context.params); - functions.logger.debug(`Upserting document ${JSON.stringify(typesenseDocument)}`); + if (config.shouldLogTypesenseInserts) { + functions.logger.debug(`Upserting document ${JSON.stringify(typesenseDocument)}`); + } else { + functions.logger.debug(`Upserting document ${typesenseDocument.id}`); + } return await typesense .collections(encodeURIComponent(config.typesenseCollectionName)) .documents() diff --git a/functions/src/utils.js b/functions/src/utils.js index b6b67ca..8d5fa32 100644 --- a/functions/src/utils.js +++ b/functions/src/utils.js @@ -145,8 +145,6 @@ exports.typesenseDocumentFromSnapshot = async (firestoreDocumentSnapshot, contex // using flat to flatten nested objects for older versions of Typesense that did not support nested fields // https://typesense.org/docs/0.22.2/api/collections.html#indexing-nested-fields const typesenseDocument = config.shouldFlattenNestedDocuments ? flattenDocument(mappedDocument) : mappedDocument; - console.log("typesenseDocument", typesenseDocument); - typesenseDocument.id = firestoreDocumentSnapshot.id; if (contextParams && Object.entries(contextParams).length) { From 919cd5372046a85978091a0e615b62d198723f29 Mon Sep 17 00:00:00 2001 From: James Sasitorn Date: Sun, 29 Dec 2024 18:35:25 -0800 Subject: [PATCH 2/3] Add test cases for logging tests using TestEnvironment class --- .eslintrc.js | 2 +- package.json | 5 +- test/support/testEnvironment.js | 239 ++++++++++++++++++++++++++++++++ test/writeLogging.spec.js | 193 ++++++++++++++++++++++++++ 4 files changed, 436 insertions(+), 3 deletions(-) create mode 100644 test/support/testEnvironment.js create mode 100644 test/writeLogging.spec.js diff --git a/.eslintrc.js b/.eslintrc.js index 62fa8ef..6917d17 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -6,7 +6,7 @@ module.exports = { jest: true, }, parserOptions: { - ecmaVersion: 2020, + ecmaVersion: 2022, }, extends: ["eslint:recommended", "google"], rules: { diff --git a/package.json b/package.json index dd5ead9..c7774d9 100644 --- a/package.json +++ b/package.json @@ -4,10 +4,11 @@ "scripts": { "emulator": "cross-env DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:start --import=emulator_data", "export": "firebase emulators:export emulator_data", - "test": "npm run test-part-1 && npm run test-part-2 && npm run test-part-3", - "test-part-1": "cp -f extensions/test-params-flatten-nested-true.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-true.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testPathIgnorePatterns=\"WithoutFlattening\" --testPathIgnorePatterns=\"Subcollection\"'", + "test": "npm run test-part-1 && npm run test-part-2 && npm run test-part-3 && npm run test-part-4", + "test-part-1": "cp -f extensions/test-params-flatten-nested-true.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-true.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"WithFlattening\" --testRegex=\"backfill.spec\"'", "test-part-2": "cp -f extensions/test-params-flatten-nested-false.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"WithoutFlattening\"'", "test-part-3": "cp -f extensions/test-params-subcategory-flatten-nested-false.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-subcategory-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"Subcollection\"'", + "test-part-4": "jest --testRegex=\"writeLogging\" --detectOpenHandles", "typesenseServer": "docker compose up", "lint:fix": "eslint . --fix", "lint": "eslint ." diff --git a/test/support/testEnvironment.js b/test/support/testEnvironment.js new file mode 100644 index 0000000..fb00b4d --- /dev/null +++ b/test/support/testEnvironment.js @@ -0,0 +1,239 @@ +const {execSync, spawn} = require("child_process"); +const firebase = require("firebase-admin"); +const path = require("path"); +const fs = require("fs"); + +const directConsole = require("console"); + +/** + * Load the default Firebase project ID from .firebaserc and set it to process.env.GCLOUD_PROJECT. + * Also set FIRESTORE_EMULATOR_HOST to the Firestore emulator host. + * @param {string} projectRootPath the path to the root of the project + * @throws {Error} - If the .firebaserc file is missing or the default project is not set. + */ +function loadFirebaseEnvironment(projectRootPath) { + try { + const firebaseRc = JSON.parse(fs.readFileSync(path.resolve(projectRootPath, ".firebaserc"), "utf8")); + const defaultProjectId = firebaseRc.projects?.default; + + if (!defaultProjectId) { + throw new Error("No default project found in .firebaserc"); + } + + process.env.GCLOUD_PROJECT = defaultProjectId; + process.env.FIRESTORE_EMULATOR_HOST = "localhost:8080"; // explcitly set emulator host, used for firebase-admin + } catch (error) { + console.error("Error loading .firebaserc:", error.message); + throw error; + } +} + +/** + * Class that sets up a test environment for the Firestore Typesense Search extension. + *

+ * Creates a new Firebase project, sets up the Firestore emulator, and + * configures the extension to use the Firestore emulator as the backend. + *

+ * Also provides a Firestore client for tests to use, and a Typesense client + * that is configured to connect to the Typesense server running on localhost. + *

+ * Additionally, provides a method to capture the emulator logs and write them + * to a file. + */ +class TestEnvironment { + // Global ouput all emulator logs + projectRootPath = path.resolve(__dirname, "../../"); + firebaseEnvPath = "extensions/firestore-typesense-search.env.local"; + shouldOutputAllEmulatorLogs = false; + dotenvPath = null; + dotenvConfig = null; + + // Emulator vars + emulator = null; + capturedEmulatorLogs = ""; + shouldLogEmulator = false; + + // Test client case vars + config = null; + firebaseApp = null; + firestore = null; + typesense = null; + + /** + * Initialize a test environment with a specific dotenv config and a flag for logging all emulator logs. + * @param {{dotenvConfig: string, debugLog: boolean}} config + * @param {string} config.dotenvConfig - path to the env file to use for the firebase emulator and test + * @param {boolean} config.debugLog - whether to log all emulator logs to console + */ + constructor({ + dotenvPath, + dotenvConfig, + outputAllEmulatorLogs = false, + } = {}) { + this.dotenvPath = dotenvPath; + this.dotenvConfig = dotenvConfig; + this.shouldOutputAllEmulatorLogs = outputAllEmulatorLogs; + + if (dotenvPath && dotenvConfig) { + throw new Error("Provide either 'dotenvPath' or 'dotenvConfig', not both."); + } + } + + /** + * Set up a test environment for the Firestore Typesense Search extension. + *

+ * Creates a new Firebase project, sets up the Firestore emulator, and + * configures the extension to use the Firestore emulator as the backend. + *

+ * Also provides a Firestore client for tests to use, and a Typesense client + * that is configured to connect to the Typesense server running on localhost. + *

+ * @param {function} done - callback to call when the test environment is set up + */ + setupTestEnvironment(done) { + directConsole.log("Setting Up Firebase emulator..."); + if (this.dotenvPath) { + directConsole.log(`Copying ${this.dotenvPath} to ${this.firebaseEnvPath}...`); + execSync( + `cp -f ${this.dotenvPath} ${this.firebaseEnvPath}`, + ); + } else if (this.dotenvConfig) { + fs.writeFileSync(this.firebaseEnvPath, this.dotenvConfig); + } + + this.emulator = spawn( + "firebase", + [ + "emulators:start", + "--only", + "functions,firestore,extensions", + ], + { + stdio: ["pipe", "pipe", "pipe"], + env: { + ...process.env, + NODE_OPTIONS: "--experimental-vm-modules", + FORCE_COLOR: "1", + }, + }, + ); + + // Listen for logs from the emulator + this.emulator.stdout.on("data", (data) => { + let logMessage = data.toString().trim(); + + if (this.shouldLogEmulator) { + try { + // eslint-disable-next-line no-control-regex + const stripAnsi = (str) => str.replace(/\x1b\[[0-9;]*m/g, ""); + let flatLogMessage = stripAnsi(logMessage); + + if (flatLogMessage.startsWith(">")) { + flatLogMessage = flatLogMessage.replace(/^>\s*/, ""); // Removes "> " + } + + const parsedLog = JSON.parse(flatLogMessage); + logMessage = parsedLog.message; + } catch (e) { + // Not a JSON log, keep the original logMessage + } + this.capturedEmulatorLogs += logMessage + "\n"; + } + + if (this.shouldOutputAllEmulatorLogs) { + directConsole.log(logMessage); + } + + if (logMessage.includes("All emulators ready")) { // Adjust to the actual readiness log message + directConsole.log("Emulator is ready"); + + try { + directConsole.log("Loading testing firebase config and modules..."); + const dotenvResult = require("dotenv").config({path: path.resolve(this.projectRootPath, this.firebaseEnvPath)}); // load same .env as emulator + if (dotenvResult.error) { + throw dotenvResult.error; + } + + this.config = require(path.resolve(this.projectRootPath, "functions/src/config.js")); + this.typesense = require(path.resolve(this.projectRootPath, "functions/src/createTypesenseClient.js"))(); + + loadFirebaseEnvironment(this.projectRootPath); + + directConsole.log("Initializing Firebase Admin Client..."); + this.firebaseApp = firebase.initializeApp({ + databaseURL: `${process.env.FIRESTORE_EMULATOR_HOST}?ns=${process.env.GCLOUD_PROJECT}`, + projectId: process.env.GCLOUD_PROJECT, + }); + this.firestore = this.firebaseApp.firestore(); + + directConsole.log("Test environment initialization complete"); + this.shouldLogEmulator = true; + done(); + } catch (e) { + directConsole.error("Error loading environment variables:", e); + done(e); + } + } + }); + + this.emulator.stderr.on("data", (data) => { + directConsole.error(data.toString()); + }); + + this.emulator.on("close", (code) => { + if (code !== 0) { + done(new Error("Emulator exited unexpectedly.")); + } + }); + } + + /** + * Reset the captured emulator logs. + * This will clear the logs collected from the Firebase Emulator. + */ + resetCapturedEmulatorLogs() { + this.capturedEmulatorLogs = ""; + } + + /** + * Clean up the test environment after all tests have run. + * This will shut down the Firebase Emulator and clean up any data that was created during the tests. + */ + async teardownTestEnvironment() { + this.shouldLogEmulator = false; + if (this.emulator) { + this.emulator.kill("SIGINT"); + await new Promise((resolve) => this.emulator.on("exit", resolve)); + } + + await this.firebaseApp.delete(); + } + + /** + * Clears all data in Firestore and Typesense before a test is run. + * This will delete the Firestore collection and any data in it, and + * delete the Typesense collection and any data in it. + */ + async clearAllData() { + try { + await this.firestore.recursiveDelete(this.firestore.collection(this.config.firestoreCollectionPath)); + } catch (e) { + directConsole.info(`${this.config.firestoreCollectionPath} collection not found, proceeding...`); + } + + try { + await this.typesense.collections(encodeURIComponent(this.config.typesenseCollectionName)).delete(); + } catch (e) { + directConsole.info(`${this.config.typesenseCollectionName} collection not found, proceeding...`); + } + await this.typesense.collections().create({ + name: this.config.typesenseCollectionName, + fields: [{name: ".*", type: "auto"}], + enable_nested_fields: true, + }); + } +} + +module.exports = { + TestEnvironment, +}; diff --git a/test/writeLogging.spec.js b/test/writeLogging.spec.js new file mode 100644 index 0000000..34ea3e2 --- /dev/null +++ b/test/writeLogging.spec.js @@ -0,0 +1,193 @@ +const {TestEnvironment} = require("./support/testEnvironment"); + +describe("indexOnWriteLogging - when shouldLogTypesenseInserts is false", () => { + let testEnvironment; + + beforeAll((done) => { + try { + testEnvironment = new TestEnvironment({ + dotenvPath: "extensions/test-params-flatten-nested-true.local.env", + }); + testEnvironment.setupTestEnvironment(done); + } catch (e) { + console.error(e); + done(e); + } + }); + + afterAll(async () => { + await testEnvironment.teardownTestEnvironment(); + }); + + beforeEach(async () => { + await testEnvironment.clearAllData(); + }); + + describe("testing onWrite logging", () => { + it("logs only itemId", async () => { + const docData = { + author: "value1", + title: "value2", + }; + testEnvironment.resetCapturedEmulatorLogs(); + const docRef = await testEnvironment.firestore.collection(testEnvironment.config.firestoreCollectionPath).add(docData); + + await new Promise((r) => setTimeout(r, 5000)); + expect(testEnvironment.capturedEmulatorLogs).toContain( + `Upserting document ${docRef.id}`, + ); + }); + }); + + describe("testing backfill logging", () => { + it("backfills existing Firestore data in all collections to Typesense", async () => { + const book = { + author: "Author A", + title: "Title X", + country: "USA", + }; + const firestoreDoc = await testEnvironment.firestore.collection(testEnvironment.config.firestoreCollectionPath).add(book); + // Wait for firestore cloud function to write to Typesense + await new Promise((r) => setTimeout(r, 2000)); + + // The above will automatically add the document to Typesense, + // so delete it so we can test backfill + await testEnvironment.typesense.collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName)).delete(); + await testEnvironment.typesense.collections().create({ + name: testEnvironment.config.typesenseCollectionName, + fields: [ + {name: ".*", type: "auto"}, + ], + }); + + await testEnvironment.firestore + .collection(testEnvironment.config.typesenseBackfillTriggerDocumentInFirestore.split("/")[0]) + .doc("backfill") + .set({trigger: true}); + // Wait for firestore cloud function to write to Typesense + await new Promise((r) => setTimeout(r, 2000)); + + // Check that the data was backfilled + const typesenseDocsStr = await testEnvironment.typesense + .collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName)) + .documents() + .export(); + const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); + expect(typesenseDocs.length).toBe(1); + expect(typesenseDocs[0]).toStrictEqual({ + id: firestoreDoc.id, + author: book.author, + title: book.title, + }); + + // Check that the backfill log was written + expect(testEnvironment.capturedEmulatorLogs).not.toContain( + "Backfilling document", + ); + + expect(testEnvironment.capturedEmulatorLogs).toContain( + "Imported 1 documents into Typesense", + ); + }); + }); +}); + +describe("indexOnWriteLogging - when shouldLogTypesenseInserts is true", () => { + let testEnvironment; + + beforeAll((done) => { + testEnvironment = new TestEnvironment({ + dotenvConfig: ` +LOCATION=us-central1 +FIRESTORE_COLLECTION_PATH=books +FIRESTORE_COLLECTION_FIELDS=author,title,rating,isAvailable,location,createdAt,nested_field,tags,nullField,ref +FLATTEN_NESTED_DOCUMENTS=true +LOG_TYPESENSE_INSERTS=true +TYPESENSE_HOSTS=localhost +TYPESENSE_PORT=8108 +TYPESENSE_PROTOCOL=http +TYPESENSE_COLLECTION_NAME=books_firestore/1 +TYPESENSE_API_KEY=xyz +`, + }); + testEnvironment.setupTestEnvironment(done); + }); + + afterAll(async () => { + await testEnvironment.teardownTestEnvironment(); + }); + + beforeEach(async () => { + await testEnvironment.clearAllData(); + }); + + describe("testing basic onWrite logging", () => { + it("logs detailed inserts", async () => { + const docData = { + author: "value1", + title: "value2", + }; + + testEnvironment.resetCapturedEmulatorLogs(); + const docRef = await testEnvironment.firestore.collection(testEnvironment.config.firestoreCollectionPath).add(docData); + + await new Promise((r) => setTimeout(r, 5000)); + expect(testEnvironment.capturedEmulatorLogs).toContain( + `Upserting document ${JSON.stringify({...docData, id: docRef.id})}`, + ); + }); + }); + + describe("testing backfill logging", () => { + it("backfills existing Firestore data in all collections to Typesense", async () => { + const book = { + author: "Author A", + title: "Title X", + country: "USA", + }; + const firestoreDoc = await testEnvironment.firestore.collection(testEnvironment.config.firestoreCollectionPath).add(book); + // Wait for firestore cloud function to write to Typesense + await new Promise((r) => setTimeout(r, 2000)); + + // The above will automatically add the document to Typesense, + // so delete it so we can test backfill + await testEnvironment.typesense.collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName)).delete(); + await testEnvironment.typesense.collections().create({ + name: testEnvironment.config.typesenseCollectionName, + fields: [ + {name: ".*", type: "auto"}, + ], + }); + + await testEnvironment.firestore + .collection(testEnvironment.config.typesenseBackfillTriggerDocumentInFirestore.split("/")[0]) + .doc("backfill") + .set({trigger: true}); + // Wait for firestore cloud function to write to Typesense + await new Promise((r) => setTimeout(r, 2000)); + + // Check that the data was backfilled + const typesenseDocsStr = await testEnvironment.typesense + .collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName)) + .documents() + .export(); + const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); + expect(typesenseDocs.length).toBe(1); + const expectedResult = { + author: book.author, + title: book.title, + id: firestoreDoc.id, + }; + expect(typesenseDocs[0]).toStrictEqual(expectedResult); + + // Check that the backfill log was written + expect(testEnvironment.capturedEmulatorLogs).toContain( + `Backfilling document ${JSON.stringify(expectedResult)}`, + ); + + expect(testEnvironment.capturedEmulatorLogs).toContain( + "Imported 1 documents into Typesense", + ); + }); + }); +}); From 181324c0a289c7f2cb803b9332ec6bb1d6936c49 Mon Sep 17 00:00:00 2001 From: James Sasitorn Date: Sun, 29 Dec 2024 20:16:58 -0800 Subject: [PATCH 3/3] Migrate Subcollection tests to use TestEnvironment and merge test-part-3 and test-part-4 --- package.json | 5 +- test/backfillSubcollection.spec.js | 111 +++++++++++++------------ test/indexOnWriteSubcollection.spec.js | 74 ++++++++--------- 3 files changed, 97 insertions(+), 93 deletions(-) diff --git a/package.json b/package.json index c7774d9..ef1d060 100644 --- a/package.json +++ b/package.json @@ -4,11 +4,10 @@ "scripts": { "emulator": "cross-env DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:start --import=emulator_data", "export": "firebase emulators:export emulator_data", - "test": "npm run test-part-1 && npm run test-part-2 && npm run test-part-3 && npm run test-part-4", + "test": "npm run test-part-1 && npm run test-part-2 && npm run test-part-3", "test-part-1": "cp -f extensions/test-params-flatten-nested-true.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-true.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"WithFlattening\" --testRegex=\"backfill.spec\"'", "test-part-2": "cp -f extensions/test-params-flatten-nested-false.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"WithoutFlattening\"'", - "test-part-3": "cp -f extensions/test-params-subcategory-flatten-nested-false.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-subcategory-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"Subcollection\"'", - "test-part-4": "jest --testRegex=\"writeLogging\" --detectOpenHandles", + "test-part-3": "jest --testRegex=\"writeLogging\" --testRegex=\"Subcollection\" --detectOpenHandles", "typesenseServer": "docker compose up", "lint:fix": "eslint . --fix", "lint": "eslint ." diff --git a/test/backfillSubcollection.spec.js b/test/backfillSubcollection.spec.js index b13870e..b71e4eb 100644 --- a/test/backfillSubcollection.spec.js +++ b/test/backfillSubcollection.spec.js @@ -1,59 +1,56 @@ -const firebase = require("firebase-admin"); -const config = require("../functions/src/config.js"); -const typesense = require("../functions/src/createTypesenseClient.js")(); - -const app = firebase.initializeApp({ - // Use a special URL to talk to the Realtime Database emulator - databaseURL: `${process.env.FIREBASE_DATABASE_EMULATOR_HOST}?ns=${process.env.GCLOUD_PROJECT}`, - projectId: process.env.GCLOUD_PROJECT, -}); -const firestore = app.firestore(); +const {TestEnvironment} = require("./support/testEnvironment"); + +// test case configs +const TEST_FIRESTORE_PARENT_COLLECTION_PATH = "users"; +const TEST_FIRESTORE_CHILD_FIELD_NAME = "books"; describe("backfillSubcollection", () => { - const parentCollectionPath = process.env.TEST_FIRESTORE_PARENT_COLLECTION_PATH; + let testEnvironment; + + const parentCollectionPath = TEST_FIRESTORE_PARENT_COLLECTION_PATH; const unrelatedCollectionPath = "unrelatedCollectionToNotBackfill"; - const childFieldName = process.env.TEST_FIRESTORE_CHILD_FIELD_NAME; + const childFieldName = TEST_FIRESTORE_CHILD_FIELD_NAME; let parentIdField = null; - beforeAll(() => { - const matches = config.firestoreCollectionPath.match(/{([^}]+)}/g); - expect(matches).toBeDefined(); - expect(matches.length).toBe(1); + let config = null; + let firestore = null; + let typesense = null; + + beforeAll((done) => { + testEnvironment = new TestEnvironment({ + dotenvPath: "extensions/test-params-subcategory-flatten-nested-false.local.env", + outputAllEmulatorLogs: true, + }); + testEnvironment.setupTestEnvironment((err) => { + const matches = testEnvironment.config.firestoreCollectionPath.match(/{([^}]+)}/g); + expect(matches).toBeDefined(); + expect(matches.length).toBe(1); + + parentIdField = matches[0].replace(/{|}/g, ""); + expect(parentIdField).toBeDefined(); + + config = testEnvironment.config; + firestore = testEnvironment.firestore; + typesense = testEnvironment.typesense; + done(); + }); + }); - parentIdField = matches[0].replace(/{|}/g, ""); - expect(parentIdField).toBeDefined(); + afterAll(async () => { + await testEnvironment.teardownTestEnvironment(); }); beforeEach(async () => { - // Clear the database between tests + // For subcollections, need special handling to clear parent collection. Deleting here triggers firebase functions await firestore.recursiveDelete(firestore.collection(parentCollectionPath)); await firestore.recursiveDelete(firestore.collection(unrelatedCollectionPath)); - // Clear any previously created collections - try { - await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete(); - } catch (e) { - console.info(`${config.typesenseCollectionName} collection not found, proceeding...`); - } - - // Create a new Typesense collection - return typesense.collections().create({ - name: config.typesenseCollectionName, - fields: [ - {name: ".*", type: "auto"}, - ], - }); - }); - - afterAll(async () => { - // clean up the firebase app after all tests have run - await app.delete(); + await testEnvironment.clearAllData(); }); describe("when firestore_collections is not specified", () => { it("backfills existing Firestore data in all collections to Typesense" + - " when `trigger: true` is set " + - ` in ${config.typesenseBackfillTriggerDocumentInFirestore}`, async () => { + " when `trigger: true` is set on trigger document", async () => { const parentDocData = { nested_field: { field1: "value1", @@ -67,7 +64,7 @@ describe("backfillSubcollection", () => { }; // create parent document in Firestore - const parentDocRef = await firestore.collection(parentCollectionPath).add(parentDocData); + const parentDocRef = await testEnvironment.firestore.collection(parentCollectionPath).add(parentDocData); // create a subcollection with document under the parent document const subCollectionRef = parentDocRef.collection(childFieldName); @@ -78,24 +75,24 @@ describe("backfillSubcollection", () => { // The above will automatically add the document to Typesense, // so delete it so we can test backfill - await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete(); - await typesense.collections().create({ - name: config.typesenseCollectionName, + await testEnvironment.typesense.collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName)).delete(); + await testEnvironment.typesense.collections().create({ + name: testEnvironment.config.typesenseCollectionName, fields: [ {name: ".*", type: "auto"}, ], }); - await firestore - .collection(config.typesenseBackfillTriggerDocumentInFirestore.split("/")[0]) + await testEnvironment.firestore + .collection(testEnvironment.config.typesenseBackfillTriggerDocumentInFirestore.split("/")[0]) .doc("backfill") .set({trigger: true}); // Wait for firestore cloud function to write to Typesense await new Promise((r) => setTimeout(r, 2000)); // Check that the data was backfilled - const typesenseDocsStr = await typesense - .collections(encodeURIComponent(config.typesenseCollectionName)) + const typesenseDocsStr = await testEnvironment.typesense + .collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName)) .documents() .export(); const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); @@ -113,8 +110,7 @@ describe("backfillSubcollection", () => { describe("when firestore_collections is specified", () => { describe("when firestore_collections includes this collection", () => { it("backfills existing Firestore data in this particular collection to Typesense" + - " when `trigger: true` is set " + - ` in ${config.typesenseBackfillTriggerDocumentInFirestore}`, async () => { + " when `trigger: true` is set on trigger document", async () => { const parentDocData = { nested_field: { field1: "value1", @@ -163,6 +159,7 @@ describe("backfillSubcollection", () => { .documents() .export(); const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); + console.log(typesenseDocs); expect(typesenseDocs.length).toBe(1); expect(typesenseDocs[0]).toStrictEqual({ id: subDocRef.id, @@ -175,8 +172,7 @@ describe("backfillSubcollection", () => { describe("when firestore_collections does not include this collection", () => { it("does not backfill existing Firestore data in this particular collection to Typesense" + - " when `trigger: true` is set " + - ` in ${config.typesenseBackfillTriggerDocumentInFirestore}`, async () => { + " when `trigger: true` is set on trigger document", async () => { const parentDocData = { nested_field: { field1: "value1", @@ -194,7 +190,7 @@ describe("backfillSubcollection", () => { // create a subcollection with document under the parent document const subCollectionRef = parentDocRef.collection(childFieldName); - await subCollectionRef.add(subDocData); + const subDocRef = await subCollectionRef.add(subDocData); // Wait for firestore cloud function to write to Typesense await new Promise((r) => setTimeout(r, 2000)); @@ -224,6 +220,15 @@ describe("backfillSubcollection", () => { .documents() .export(); expect(typesenseDocsStr).toEqual(""); + + // Check that the error was logged + testEnvironment.resetCapturedEmulatorLogs(); + subDocRef.delete(); + await new Promise((r) => setTimeout(r, 5000)); + + expect(testEnvironment.capturedEmulatorLogs).toContain( + `Could not find a document with id: ${subDocRef.id}`, + ); }); }); }); diff --git a/test/indexOnWriteSubcollection.spec.js b/test/indexOnWriteSubcollection.spec.js index 806f546..a6908a0 100644 --- a/test/indexOnWriteSubcollection.spec.js +++ b/test/indexOnWriteSubcollection.spec.js @@ -1,50 +1,50 @@ -const firebase = require("firebase-admin"); -const config = require("../functions/src/config.js"); -const typesense = require("../functions/src/createTypesenseClient.js")(); - -const app = firebase.initializeApp({ - // Use a special URL to talk to the Realtime Database emulator - databaseURL: `${process.env.FIREBASE_DATABASE_EMULATOR_HOST}?ns=${process.env.GCLOUD_PROJECT}`, - projectId: process.env.GCLOUD_PROJECT, -}); -const firestore = app.firestore(); +const {TestEnvironment} = require("./support/testEnvironment"); + +// test case configs +const TEST_FIRESTORE_PARENT_COLLECTION_PATH = "users"; +const TEST_FIRESTORE_CHILD_FIELD_NAME = "books"; + describe("indexOnWriteSubcollection", () => { - const parentCollectionPath = process.env.TEST_FIRESTORE_PARENT_COLLECTION_PATH; - const childFieldName = process.env.TEST_FIRESTORE_CHILD_FIELD_NAME; + let testEnvironment; + + const parentCollectionPath = TEST_FIRESTORE_PARENT_COLLECTION_PATH; + const childFieldName = TEST_FIRESTORE_CHILD_FIELD_NAME; let parentIdField = null; - beforeAll(async () => { - const matches = config.firestoreCollectionPath.match(/{([^}]+)}/g); - expect(matches).toBeDefined(); - expect(matches.length).toBe(1); + let config = null; + let firestore = null; + let typesense = null; - parentIdField = matches[0].replace(/{|}/g, ""); - expect(parentIdField).toBeDefined(); + beforeAll((done) => { + testEnvironment = new TestEnvironment({ + dotenvPath: "extensions/test-params-subcategory-flatten-nested-false.local.env", + outputAllEmulatorLogs: true, + }); + testEnvironment.setupTestEnvironment((err) => { + const matches = testEnvironment.config.firestoreCollectionPath.match(/{([^}]+)}/g); + expect(matches).toBeDefined(); + expect(matches.length).toBe(1); + + parentIdField = matches[0].replace(/{|}/g, ""); + expect(parentIdField).toBeDefined(); + + config = testEnvironment.config; + firestore = testEnvironment.firestore; + typesense = testEnvironment.typesense; + done(); + }); + }); + + afterAll(async () => { + await testEnvironment.teardownTestEnvironment(); }); beforeEach(async () => { - // delete the Firestore collection + // For subcollections, need special handling to clear parent collection. Deleting here triggers firebase functions await firestore.recursiveDelete(firestore.collection(parentCollectionPath)); - // Clear any previously created collections - try { - await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete(); - } catch (e) { - console.info(`${config.typesenseCollectionName} collection not found, proceeding...`); - } - - // recreate the Typesense collection - await typesense.collections().create({ - name: config.typesenseCollectionName, - fields: [{name: ".*", type: "auto"}], - enable_nested_fields: true, - }); - }); - - afterAll(async () => { - // clean up the whole firebase app - await app.delete(); + await testEnvironment.clearAllData(); }); describe("Backfill from dynamic subcollections", () => {