From 1066c3c1f690332263b5e502070050845e8b0ed8 Mon Sep 17 00:00:00 2001 From: branberry Date: Sat, 7 Jun 2025 10:36:41 -0500 Subject: [PATCH 1/8] create docker compose file --- compose.yaml | 23 +++++++++++++++++++++++ src/Marian/index.ts | 1 + src/SearchIndex/index.ts | 1 - src/index.ts | 1 + 4 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 compose.yaml diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..56fe41d --- /dev/null +++ b/compose.yaml @@ -0,0 +1,23 @@ +services: + transport: + build: . + ports: + - "8000:8080" + env_file: .env + environment: + ATLAS_URI: "mongodb://user:pass@atlas:27019/?directConnection=true" + POOL_ATLAS_URI: "mongodb://user:pass@atlas:27019/?directConnection=true" + depends_on: + - atlas + develop: + watch: + - action: sync + path: . + target: /src + atlas: + image: mongodb/mongodb-atlas-local:latest + ports: + - 27019:27017 + environment: + - MONGO_INITDB_ROOT_USERNAME=user + - MONGO_INITDB_ROOT_PASSWORD=pass diff --git a/src/Marian/index.ts b/src/Marian/index.ts index 38979fd..b3452c4 100644 --- a/src/Marian/index.ts +++ b/src/Marian/index.ts @@ -268,6 +268,7 @@ export default class Marian { } const res = await fetch(url); const toml = await res.text(); + return parse(toml); } catch (e) { log.error(`Error while fetching taxonomy: ${JSON.stringify(e)}`); diff --git a/src/SearchIndex/index.ts b/src/SearchIndex/index.ts index c7ca448..fc1c1f0 100644 --- a/src/SearchIndex/index.ts +++ b/src/SearchIndex/index.ts @@ -4,7 +4,6 @@ import Logger from 'basic-logger'; import { MongoClient, Collection, TransactionOptions, AnyBulkWriteOperation, Db, ClientSession, Filter } from 'mongodb'; import { - compareFacets, convertTaxonomyToResponseFormat, convertTaxonomyToTrie, formatFacetMetaResponse, diff --git a/src/index.ts b/src/index.ts index 9560d76..4179494 100755 --- a/src/index.ts +++ b/src/index.ts @@ -53,6 +53,7 @@ function verifyAndGetEnvVars() { const adminPrivKey = process.env[ADMIN_API_KEY]; const taxonomyUrl = process.env[TAXONOMY_URL]; + if (!manifestUri || !s3Bucket || !atlasUri || !groupId || !adminPrivKey || !adminPubKey || !s3Path) { if (!manifestUri) { console.error(`Missing ${MANIFEST_URI_KEY}`); From e247f82a9b7b646a82b939bedcff93a3887310e4 Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 10 Jun 2025 08:57:55 -0500 Subject: [PATCH 2/8] get docker compose working --- .dockerignore | 3 ++- Dockerfile | 2 +- compose.yaml | 17 ++++++++++++----- scripts/entrypoint.sh | 0 src/SearchPropertyMapping/index.ts | 2 ++ src/index.ts | 4 +++- 6 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 scripts/entrypoint.sh diff --git a/.dockerignore b/.dockerignore index eefeee3..44746f6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,4 +5,5 @@ !package-lock.json !tsconfig.json !src/ -!resources/ \ No newline at end of file +!resources/ +!mongodb-org-8.0.0.repo \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index e44dc8f..64470e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,4 +18,4 @@ COPY --from=builder /app/build ./build COPY --from=builder /app/resources ./resources EXPOSE 8080 -ENTRYPOINT ["node", "build/index.js", "--load-manifests"] \ No newline at end of file +ENTRYPOINT ["node", "build/index.js", "--create-indexes" ,"--load-manifests"] \ No newline at end of file diff --git a/compose.yaml b/compose.yaml index 56fe41d..e5cb2c7 100644 --- a/compose.yaml +++ b/compose.yaml @@ -3,21 +3,28 @@ services: build: . ports: - "8000:8080" - env_file: .env - environment: - ATLAS_URI: "mongodb://user:pass@atlas:27019/?directConnection=true" - POOL_ATLAS_URI: "mongodb://user:pass@atlas:27019/?directConnection=true" depends_on: - atlas + env_file: .env + environment: + ATLAS_URI: "mongodb://atlas:27017/?directConnection=true" + POOL_ATLAS_URI: "mongodb://atlas:27017/?directConnection=true" + develop: watch: - action: sync path: . target: /src atlas: - image: mongodb/mongodb-atlas-local:latest + image: mongodb/mongodb-atlas-local ports: - 27019:27017 environment: - MONGO_INITDB_ROOT_USERNAME=user - MONGO_INITDB_ROOT_PASSWORD=pass + # healthcheck: + # test: echo 'db.runCommand("ping").ok' | mongo mongo:27017/test --quiet + # interval: 10s + # timeout: 10s + # retries: 5 + # start_period: 40s diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh new file mode 100644 index 0000000..e69de29 diff --git a/src/SearchPropertyMapping/index.ts b/src/SearchPropertyMapping/index.ts index fac5edb..075f559 100644 --- a/src/SearchPropertyMapping/index.ts +++ b/src/SearchPropertyMapping/index.ts @@ -65,6 +65,8 @@ export const setPropertyMapping = async function () { const dbName = process.env['POOL_DB'] ?? 'pool_test'; const poolAtlasUri = verifyAndGetEnvVars(); + + console.log('poolAtlasUri',poolAtlasUri) const client = await MongoClient.connect(poolAtlasUri); const db = client.db(dbName); const collection = db.collection(collectionName); diff --git a/src/index.ts b/src/index.ts index 4179494..7ef9ce7 100755 --- a/src/index.ts +++ b/src/index.ts @@ -112,10 +112,12 @@ async function main() { let databaseName = DEFAULT_DATABASE_NAME; const envDBName = process.env[DATABASE_NAME_KEY]; if (envDBName) { - databaseName = envDBName; + databaseName = envDBName; } + console.log('atlasUri',atlasUri) const client = await MongoClient.connect(atlasUri); + console.log('connected') const searchIndex = new SearchIndex(manifestUri, s3Bucket, s3Path, client, databaseName); if (process.argv.includes('--create-indexes')) { From f276474decd6f21a4afad7d987d607c670a12085 Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 15 Jul 2025 15:59:10 -0500 Subject: [PATCH 3/8] Use constant instead of boost --- src/Query/index.ts | 9 +++++---- src/Query/types.ts | 5 ++++- src/SearchPropertyMapping/index.ts | 1 - src/data/term-result-mappings.ts | 24 +++++++++++++----------- src/index.ts | 5 +---- 5 files changed, 23 insertions(+), 21 deletions(-) diff --git a/src/Query/index.ts b/src/Query/index.ts index a7f1509..14a8ce9 100644 --- a/src/Query/index.ts +++ b/src/Query/index.ts @@ -121,15 +121,16 @@ export class Query { // if we need to boost for matching slug on an exact rawQuery match const boostedStrings = strippedMapping[this.rawQuery.trim()]; - if (Array.isArray(boostedStrings) && typeof boostedStrings[0] === 'string') { + console.log(boostedStrings); + if (Array.isArray(boostedStrings)) { parts.push( ...boostedStrings.map((boostedString, i) => ({ text: { path: 'strippedSlug', query: [boostedString], - // Boost each entry slightly higher than the next so that entry - // order is respected in results - score: { boost: { value: 100 + 10 * (boostedStrings.length - i) } }, + // Use a constant score value to guarantee that we receive the order + // that we are expecting from the term-result-mappings object. + score: { constant: { value: 200 ** (boostedStrings.length - i) } }, }, })) ); diff --git a/src/Query/types.ts b/src/Query/types.ts index 0187b5d..84d11b3 100644 --- a/src/Query/types.ts +++ b/src/Query/types.ts @@ -8,7 +8,10 @@ type PathObj = { type Path = string | PathObj; export type Score = { - boost: { + boost?: { + value: number; + }; + constant?: { value: number; }; }; diff --git a/src/SearchPropertyMapping/index.ts b/src/SearchPropertyMapping/index.ts index 075f559..380d48c 100644 --- a/src/SearchPropertyMapping/index.ts +++ b/src/SearchPropertyMapping/index.ts @@ -66,7 +66,6 @@ export const setPropertyMapping = async function () { const poolAtlasUri = verifyAndGetEnvVars(); - console.log('poolAtlasUri',poolAtlasUri) const client = await MongoClient.connect(poolAtlasUri); const db = client.db(dbName); const collection = db.collection(collectionName); diff --git a/src/data/term-result-mappings.ts b/src/data/term-result-mappings.ts index 58ee288..878c8fe 100644 --- a/src/data/term-result-mappings.ts +++ b/src/data/term-result-mappings.ts @@ -102,9 +102,17 @@ const resultMapping: ResultMapping = { 'reference/sql-aggregation-comparison', ], operator: ['reference/operator/query', 'reference/operator/update', 'reference/operator/aggregation-pipeline'], - exists: ['reference/operator/query/exists', 'atlas/atlas-search/exists', 'drivers/node/current/crud/query/query-document/'], - $exists: ['reference/operator/query/exists', 'atlas/atlas-search/exists', 'drivers/node/current/crud/query/query-document/'], - exist: ['reference/operator/query/exists', 'atlas/atlas-search/exists', 'drivers/node/current/crud/query/query-document/'], + exists: [ + 'reference/operator/query/exists', + 'atlas/atlas-search/exists', + 'drivers/node/current/crud/query/query-document/', + ], + $exists: ['reference/operator/query/exists', '/atlas-search/exists', 'crud/query/query-document/'], + exist: [ + 'reference/operator/query/exists', + 'atlas/atlas-search/exists', + 'drivers/node/current/crud/query/query-document/', + ], mongoclient: [ 'drivers/kotlin/coroutine/current/fundamentals/connection/mongoclientsettings', 'drivers/java/sync/current/connection/mongoclient', @@ -151,10 +159,7 @@ const resultMapping: ResultMapping = { $regex: ['reference/operator/query/regex', 'atlas/atlas-search/regex', 'reference/operator/aggregation/regexMatch'], $eq: ['reference/operator/query/eq', 'reference/operator/aggregation/eq', 'reference/operator/query/elemMatch'], $size: ['reference/operator/query/size', 'reference/operator/aggregation/size', 'tutorial/query-arrays'], - $sort: [ - 'reference/operator/update/sort', - 'reference/operator/aggregation/sort', - ], + $sort: ['reference/operator/update/sort', 'reference/operator/aggregation/sort'], $inc: [ 'reference/operator/update/inc', 'reference/method/db.collection.updateMany', @@ -178,10 +183,7 @@ const resultMapping: ResultMapping = { 'tutorial/update-documents-with-aggregation-pipeline', 'reference/operator/aggregation/sum', ], - $first: [ - 'reference/operator/aggregation/first', - 'reference/operator/aggregation/firstN', - ], + $first: ['reference/operator/aggregation/first', 'reference/operator/aggregation/firstN'], $count: ['reference/operator/aggregation/count', 'reference/command/count', 'atlas/atlas-search/counting'], indexes: ['indexes'], $addFields: [ diff --git a/src/index.ts b/src/index.ts index 7ef9ce7..9560d76 100755 --- a/src/index.ts +++ b/src/index.ts @@ -53,7 +53,6 @@ function verifyAndGetEnvVars() { const adminPrivKey = process.env[ADMIN_API_KEY]; const taxonomyUrl = process.env[TAXONOMY_URL]; - if (!manifestUri || !s3Bucket || !atlasUri || !groupId || !adminPrivKey || !adminPubKey || !s3Path) { if (!manifestUri) { console.error(`Missing ${MANIFEST_URI_KEY}`); @@ -112,12 +111,10 @@ async function main() { let databaseName = DEFAULT_DATABASE_NAME; const envDBName = process.env[DATABASE_NAME_KEY]; if (envDBName) { - databaseName = envDBName; + databaseName = envDBName; } - console.log('atlasUri',atlasUri) const client = await MongoClient.connect(atlasUri); - console.log('connected') const searchIndex = new SearchIndex(manifestUri, s3Bucket, s3Path, client, databaseName); if (process.argv.includes('--create-indexes')) { From a4c5e39f842869a169b58c0a9b4de41e37d0d341 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 17 Jul 2025 10:52:33 -0500 Subject: [PATCH 4/8] fix paths --- src/data/term-result-mappings.ts | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/data/term-result-mappings.ts b/src/data/term-result-mappings.ts index 878c8fe..237f998 100644 --- a/src/data/term-result-mappings.ts +++ b/src/data/term-result-mappings.ts @@ -102,17 +102,9 @@ const resultMapping: ResultMapping = { 'reference/sql-aggregation-comparison', ], operator: ['reference/operator/query', 'reference/operator/update', 'reference/operator/aggregation-pipeline'], - exists: [ - 'reference/operator/query/exists', - 'atlas/atlas-search/exists', - 'drivers/node/current/crud/query/query-document/', - ], - $exists: ['reference/operator/query/exists', '/atlas-search/exists', 'crud/query/query-document/'], - exist: [ - 'reference/operator/query/exists', - 'atlas/atlas-search/exists', - 'drivers/node/current/crud/query/query-document/', - ], + exists: ['reference/operator/query/exists', 'atlas-search/exists', 'drivers/node/current/crud/query/query-document/'], + $exists: ['reference/operator/query/exists', 'atlas-search/exists', 'crud/query/query-document/'], + exist: ['reference/operator/query/exists', 'atlas-search/exists', 'drivers/node/current/crud/query/query-document/'], mongoclient: [ 'drivers/kotlin/coroutine/current/fundamentals/connection/mongoclientsettings', 'drivers/java/sync/current/connection/mongoclient', From 725f3c1bd156ef223e474e0053441a87dfb35352 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 17 Jul 2025 11:04:57 -0500 Subject: [PATCH 5/8] remove unnecessary changes --- .dockerignore | 3 +-- Dockerfile | 2 +- compose.yaml | 30 ------------------------------ 3 files changed, 2 insertions(+), 33 deletions(-) delete mode 100644 compose.yaml diff --git a/.dockerignore b/.dockerignore index 44746f6..eefeee3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,5 +5,4 @@ !package-lock.json !tsconfig.json !src/ -!resources/ -!mongodb-org-8.0.0.repo \ No newline at end of file +!resources/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 64470e0..e44dc8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,4 +18,4 @@ COPY --from=builder /app/build ./build COPY --from=builder /app/resources ./resources EXPOSE 8080 -ENTRYPOINT ["node", "build/index.js", "--create-indexes" ,"--load-manifests"] \ No newline at end of file +ENTRYPOINT ["node", "build/index.js", "--load-manifests"] \ No newline at end of file diff --git a/compose.yaml b/compose.yaml deleted file mode 100644 index e5cb2c7..0000000 --- a/compose.yaml +++ /dev/null @@ -1,30 +0,0 @@ -services: - transport: - build: . - ports: - - "8000:8080" - depends_on: - - atlas - env_file: .env - environment: - ATLAS_URI: "mongodb://atlas:27017/?directConnection=true" - POOL_ATLAS_URI: "mongodb://atlas:27017/?directConnection=true" - - develop: - watch: - - action: sync - path: . - target: /src - atlas: - image: mongodb/mongodb-atlas-local - ports: - - 27019:27017 - environment: - - MONGO_INITDB_ROOT_USERNAME=user - - MONGO_INITDB_ROOT_PASSWORD=pass - # healthcheck: - # test: echo 'db.runCommand("ping").ok' | mongo mongo:27017/test --quiet - # interval: 10s - # timeout: 10s - # retries: 5 - # start_period: 40s From e478f2210a42dbdc8230291e8c50f63fd072cf02 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 17 Jul 2025 11:05:19 -0500 Subject: [PATCH 6/8] remove unnecessary changes --- scripts/entrypoint.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 scripts/entrypoint.sh diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh deleted file mode 100644 index e69de29..0000000 From c28ac77222448d4075bdd419c7bef13e363f99ae Mon Sep 17 00:00:00 2001 From: branberry Date: Mon, 21 Jul 2025 14:30:17 -0500 Subject: [PATCH 7/8] refactor compound statement --- src/Query/index.ts | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/Query/index.ts b/src/Query/index.ts index 14a8ce9..b9724de 100644 --- a/src/Query/index.ts +++ b/src/Query/index.ts @@ -34,8 +34,8 @@ function constructBuryOperators(parts: Part[]): CompoundPart[] { //if given query matches a "part" result not in BURIED_PROPERTY(ex: Realm) docs, score remains unaffected { compound: { - must: [part], - mustNot: [ + must: [ + part, { text: { query: BURIED_PROPERTIES, @@ -43,13 +43,7 @@ function constructBuryOperators(parts: Part[]): CompoundPart[] { }, }, ], - }, - }, - //if given query matches a "part" result in BURIED_PROPERTY(ex: Realm) docs, bury that result - { - compound: { - must: [ - part, + mustNot: [ { text: { query: BURIED_PROPERTIES, @@ -57,6 +51,13 @@ function constructBuryOperators(parts: Part[]): CompoundPart[] { }, }, ], + }, + }, + //if given query matches a "part" result in BURIED_PROPERTY(ex: Realm) docs, bury that result + { + text: { + query: BURIED_PROPERTIES, + path: 'searchProperty', score: { boost: { value: BURIED_FACTOR } }, }, } @@ -121,7 +122,7 @@ export class Query { // if we need to boost for matching slug on an exact rawQuery match const boostedStrings = strippedMapping[this.rawQuery.trim()]; - console.log(boostedStrings); + if (Array.isArray(boostedStrings)) { parts.push( ...boostedStrings.map((boostedString, i) => ({ From fef104b65aa1fff8aaf6c7da5325583dee0e1d8d Mon Sep 17 00:00:00 2001 From: branberry Date: Mon, 21 Jul 2025 14:32:02 -0500 Subject: [PATCH 8/8] add comments --- src/Query/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Query/index.ts b/src/Query/index.ts index b9724de..1f0d1e6 100644 --- a/src/Query/index.ts +++ b/src/Query/index.ts @@ -43,6 +43,8 @@ function constructBuryOperators(parts: Part[]): CompoundPart[] { }, }, ], + // DOP-5772: Removed the part from the mustNot, and removed the second compound statement + // as that was interfering with the search results by surfacing pages we do not want. mustNot: [ { text: {