From fdc72c2bf1498da534d29913272e5289f1cdb9c2 Mon Sep 17 00:00:00 2001 From: sajeetharan Date: Wed, 29 Jul 2026 17:08:17 +0530 Subject: [PATCH 1/3] feat: add vector search, hybrid search, and search API - Setup script for Cosmos DB vector + full-text indexes - Embedding generation script (Azure OpenAI text-embedding-3-small) - Search CLI tool with vector/text/hybrid modes using RRF ranking - /api/search endpoint for frontend integration - Server-side search with CONTAINS fallback - Safe setup: validates capability before deleting container --- api/search.js | 131 ++++++++++++++++++++++++++ scripts/container-arm.json | 39 ++++++++ scripts/fulltext-policy.json | 1 + scripts/generate-embeddings.js | 112 ++++++++++++++++++++++ scripts/indexing-policy.json | 8 ++ scripts/search-developers.js | 165 +++++++++++++++++++++++++++++++++ scripts/setup-vector-search.js | 118 +++++++++++++++++++++++ scripts/vector-policy.json | 1 + server.js | 41 ++++++++ 9 files changed, 616 insertions(+) create mode 100644 api/search.js create mode 100644 scripts/container-arm.json create mode 100644 scripts/fulltext-policy.json create mode 100644 scripts/generate-embeddings.js create mode 100644 scripts/indexing-policy.json create mode 100644 scripts/search-developers.js create mode 100644 scripts/setup-vector-search.js create mode 100644 scripts/vector-policy.json diff --git a/api/search.js b/api/search.js new file mode 100644 index 0000000..e630b85 --- /dev/null +++ b/api/search.js @@ -0,0 +1,131 @@ +/** + * Vercel Serverless Function ā€” Hybrid search for developers + * + * Endpoint: /api/search?q=machine+learning+python&mode=hybrid + * + * Modes: vector | text | hybrid (default) + */ +import { CosmosClient } from '@azure/cosmos'; + +const COSMOS_ENDPOINT = process.env.COSMOS_ENDPOINT; +const COSMOS_KEY = process.env.COSMOS_KEY; +const OPENAI_ENDPOINT = process.env.AZURE_OPENAI_ENDPOINT; +const OPENAI_KEY = process.env.AZURE_OPENAI_KEY; +const EMBEDDING_DEPLOYMENT = process.env.EMBEDDING_DEPLOYMENT || 'text-embedding-3-small'; + +const DATABASE = 'devglobe'; +const CONTAINER = 'developers'; + +async function getEmbedding(text) { + const url = `${OPENAI_ENDPOINT}/openai/deployments/${EMBEDDING_DEPLOYMENT}/embeddings?api-version=2024-02-01`; + const res = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'api-key': OPENAI_KEY }, + body: JSON.stringify({ input: [text] }) + }); + const data = await res.json(); + return data.data[0].embedding; +} + +export default async function handler(req, res) { + res.setHeader('Access-Control-Allow-Origin', '*'); + res.setHeader('Content-Type', 'application/json'); + + const { q, mode = 'hybrid', top = '10' } = req.query; + + if (!q) { + return res.status(400).json({ error: 'Query parameter "q" is required' }); + } + + if (!COSMOS_ENDPOINT || !COSMOS_KEY) { + return res.status(500).json({ error: 'Cosmos DB not configured' }); + } + + try { + const client = new CosmosClient({ endpoint: COSMOS_ENDPOINT, key: COSMOS_KEY }); + const container = client.database(DATABASE).container(CONTAINER); + const limit = Math.min(parseInt(top), 50); + + let results; + + if (mode === 'vector') { + if (!OPENAI_ENDPOINT || !OPENAI_KEY) { + return res.status(500).json({ error: 'OpenAI not configured for vector search' }); + } + const embedding = await getEmbedding(q); + const { resources } = await container.items.query({ + query: ` + SELECT TOP ${limit} + c.id, c.login, c.name, c.avatarUrl, c.location, c.lat, c.lng, + c.topLanguage, c.score, c.totalStars, c.followers, c.soReputation, + VectorDistance(c.embedding, @embedding) AS relevance + FROM c + ORDER BY VectorDistance(c.embedding, @embedding) + `, + parameters: [{ name: '@embedding', value: embedding }] + }).fetchAll(); + results = resources; + + } else if (mode === 'text') { + const { resources } = await container.items.query({ + query: ` + SELECT TOP ${limit} + c.id, c.login, c.name, c.avatarUrl, c.location, c.lat, c.lng, + c.topLanguage, c.score, c.totalStars, c.followers, c.soReputation + FROM c + WHERE FullTextContains(c.login, @q) + OR FullTextContains(c.name, @q) + OR FullTextContains(c.location, @q) + OR FullTextContains(c.bio, @q) + OR FullTextContains(c.topLanguage, @q) + ORDER BY RANK FullTextScore(c.login, [@q]) + + FullTextScore(c.name, [@q]) + + FullTextScore(c.location, [@q]) + + FullTextScore(c.bio, [@q]) + + FullTextScore(c.topLanguage, [@q]) + `, + parameters: [{ name: '@q', value: q }] + }).fetchAll(); + results = resources; + + } else { + // Hybrid: RRF fusion of vector + full-text + if (!OPENAI_ENDPOINT || !OPENAI_KEY) { + return res.status(500).json({ error: 'OpenAI not configured for hybrid search' }); + } + const embedding = await getEmbedding(q); + const { resources } = await container.items.query({ + query: ` + SELECT TOP ${limit} + c.id, c.login, c.name, c.avatarUrl, c.location, c.lat, c.lng, + c.topLanguage, c.score, c.totalStars, c.followers, c.soReputation + FROM c + WHERE FullTextContains(c.login, @q) + OR FullTextContains(c.name, @q) + OR FullTextContains(c.location, @q) + OR FullTextContains(c.bio, @q) + OR FullTextContains(c.topLanguage, @q) + OR VectorDistance(c.embedding, @embedding) > 0.7 + ORDER BY RANK RRF( + FullTextScore(c.login, [@q]) + + FullTextScore(c.name, [@q]) + + FullTextScore(c.location, [@q]) + + FullTextScore(c.bio, [@q]) + + FullTextScore(c.topLanguage, [@q]), + VectorDistance(c.embedding, @embedding) + ) + `, + parameters: [ + { name: '@q', value: q }, + { name: '@embedding', value: embedding } + ] + }).fetchAll(); + results = resources; + } + + res.json({ query: q, mode, count: results.length, results }); + } catch (err) { + console.error('Search error:', err.message); + res.status(500).json({ error: 'Search failed' }); + } +} diff --git a/scripts/container-arm.json b/scripts/container-arm.json new file mode 100644 index 0000000..96a1834 --- /dev/null +++ b/scripts/container-arm.json @@ -0,0 +1,39 @@ +{ + "properties": { + "resource": { + "id": "developers", + "partitionKey": { + "paths": ["/location"], + "kind": "Hash" + }, + "indexingPolicy": { + "indexingMode": "consistent", + "automatic": true, + "includedPaths": [{"path": "/*"}], + "excludedPaths": [{"path": "/embedding/*"}, {"path": "/\"_etag\"/?"}], + "vectorIndexes": [{"path": "/embedding", "type": "quantizedFlat"}], + "fullTextIndexes": [{"path": "/login"}, {"path": "/name"}, {"path": "/location"}, {"path": "/bio"}, {"path": "/topLanguage"}] + }, + "vectorEmbeddingPolicy": { + "vectorEmbeddings": [ + { + "path": "/embedding", + "dataType": "float32", + "dimensions": 1536, + "distanceFunction": "cosine" + } + ] + }, + "fullTextPolicy": { + "defaultLanguage": "en-US", + "fullTextPaths": [ + {"path": "/login", "language": "en-US"}, + {"path": "/name", "language": "en-US"}, + {"path": "/location", "language": "en-US"}, + {"path": "/bio", "language": "en-US"}, + {"path": "/topLanguage", "language": "en-US"} + ] + } + } + } +} diff --git a/scripts/fulltext-policy.json b/scripts/fulltext-policy.json new file mode 100644 index 0000000..4958a74 --- /dev/null +++ b/scripts/fulltext-policy.json @@ -0,0 +1 @@ +{"defaultLanguage":"en-US","fullTextPaths":[{"path":"/login","language":"en-US"},{"path":"/name","language":"en-US"},{"path":"/location","language":"en-US"},{"path":"/bio","language":"en-US"},{"path":"/topLanguage","language":"en-US"}]} diff --git a/scripts/generate-embeddings.js b/scripts/generate-embeddings.js new file mode 100644 index 0000000..7885b1c --- /dev/null +++ b/scripts/generate-embeddings.js @@ -0,0 +1,112 @@ +/** + * Generate vector embeddings for developers and upload to Cosmos DB + * + * Usage: node scripts/generate-embeddings.js + * + * Uses Azure OpenAI text-embedding-3-small (1536 dimensions) + * Reads existing docs from Cosmos DB, generates embeddings, patches them back + */ +import 'dotenv/config'; +import { CosmosClient } from '@azure/cosmos'; + +const COSMOS_ENDPOINT = process.env.COSMOS_ENDPOINT; +const COSMOS_KEY = process.env.COSMOS_KEY; +const OPENAI_ENDPOINT = process.env.AZURE_OPENAI_ENDPOINT; // e.g., https://your-resource.openai.azure.com/ +const OPENAI_KEY = process.env.AZURE_OPENAI_KEY; +const EMBEDDING_DEPLOYMENT = process.env.EMBEDDING_DEPLOYMENT || 'text-embedding-3-small'; + +const DATABASE_NAME = 'devglobe'; +const CONTAINER_NAME = 'developers'; +const BATCH_SIZE = 100; // OpenAI supports up to 2048 inputs per request + +if (!OPENAI_ENDPOINT || !OPENAI_KEY) { + console.error('Required: AZURE_OPENAI_ENDPOINT and AZURE_OPENAI_KEY in .env'); + process.exit(1); +} + +/** + * Create a searchable text representation of a developer + * This is what gets embedded into vector space + */ +function buildEmbeddingText(dev) { + const parts = [ + dev.login, + dev.name || '', + dev.location || '', + dev.bio || '', + dev.topLanguage ? `Primary language: ${dev.topLanguage}` : '', + dev.totalStars > 1000 ? `${dev.totalStars} stars` : '', + dev.soReputation > 1000 ? `StackOverflow reputation: ${dev.soReputation}` : '', + dev.topRepos ? dev.topRepos.map(r => r.name).join(' ') : '' + ]; + return parts.filter(Boolean).join(' | '); +} + +/** + * Call Azure OpenAI embeddings API + */ +async function getEmbeddings(texts) { + const url = `${OPENAI_ENDPOINT}/openai/deployments/${EMBEDDING_DEPLOYMENT}/embeddings?api-version=2024-02-01`; + + const response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'api-key': OPENAI_KEY + }, + body: JSON.stringify({ input: texts }) + }); + + if (!response.ok) { + const err = await response.text(); + throw new Error(`OpenAI API error ${response.status}: ${err}`); + } + + const data = await response.json(); + return data.data.map(d => d.embedding); +} + +async function main() { + const client = new CosmosClient({ endpoint: COSMOS_ENDPOINT, key: COSMOS_KEY }); + const container = client.database(DATABASE_NAME).container(CONTAINER_NAME); + + console.log('šŸ“Š Generating embeddings for developers...\n'); + + // Fetch all developers without embeddings + const { resources: developers } = await container.items + .query('SELECT c.id, c.login, c.name, c.location, c.bio, c.topLanguage, c.totalStars, c.soReputation, c.topRepos FROM c WHERE NOT IS_DEFINED(c.embedding)') + .fetchAll(); + + console.log(` Found ${developers.length} developers needing embeddings\n`); + + let processed = 0; + for (let i = 0; i < developers.length; i += BATCH_SIZE) { + const batch = developers.slice(i, i + BATCH_SIZE); + const texts = batch.map(buildEmbeddingText); + + // Generate embeddings + const embeddings = await getEmbeddings(texts); + + // Patch each document with its embedding + for (let j = 0; j < batch.length; j++) { + const dev = batch[j]; + await container.item(dev.id, dev.location || '').patch({ + operations: [ + { op: 'add', path: '/embedding', value: embeddings[j] } + ] + }); + } + + processed += batch.length; + console.log(` Embedded: ${processed}/${developers.length}`); + + // Rate limit: ~3 requests/sec for embedding API + if (i + BATCH_SIZE < developers.length) { + await new Promise(r => setTimeout(r, 400)); + } + } + + console.log(`\nāœ… Done! ${processed} developers now have vector embeddings.`); +} + +main().catch(err => { console.error(err); process.exit(1); }); diff --git a/scripts/indexing-policy.json b/scripts/indexing-policy.json new file mode 100644 index 0000000..19b8f01 --- /dev/null +++ b/scripts/indexing-policy.json @@ -0,0 +1,8 @@ +{ + "indexingMode": "consistent", + "automatic": true, + "includedPaths": [{"path": "/*"}], + "excludedPaths": [{"path": "/embedding/*"}, {"path": "/\"_etag\"/?"}], + "vectorIndexes": [{"path": "/embedding", "type": "quantizedFlat"}], + "fullTextIndexes": [{"path": "/login"}, {"path": "/name"}, {"path": "/location"}, {"path": "/bio"}, {"path": "/topLanguage"}] +} diff --git a/scripts/search-developers.js b/scripts/search-developers.js new file mode 100644 index 0000000..a9a28d0 --- /dev/null +++ b/scripts/search-developers.js @@ -0,0 +1,165 @@ +/** + * Search developers using Cosmos DB Vector Search + Hybrid Search + * + * Usage: + * node scripts/search-developers.js "machine learning Python expert in Berlin" + * node scripts/search-developers.js "React TypeScript frontend" --mode=hybrid + * node scripts/search-developers.js "kubernetes" --mode=text + * + * Modes: + * --mode=vector ā€” Pure vector (semantic) search + * --mode=text ā€” Pure full-text (BM25) search + * --mode=hybrid ā€” Combined vector + full-text with RRF ranking (default) + */ +import 'dotenv/config'; +import { CosmosClient } from '@azure/cosmos'; + +const COSMOS_ENDPOINT = process.env.COSMOS_ENDPOINT; +const COSMOS_KEY = process.env.COSMOS_KEY; +const OPENAI_ENDPOINT = process.env.AZURE_OPENAI_ENDPOINT; +const OPENAI_KEY = process.env.AZURE_OPENAI_KEY; +const EMBEDDING_DEPLOYMENT = process.env.EMBEDDING_DEPLOYMENT || 'text-embedding-3-small'; + +const DATABASE_NAME = 'devglobe'; +const CONTAINER_NAME = 'developers'; + +// Parse args +const args = process.argv.slice(2); +const modeArg = args.find(a => a.startsWith('--mode=')); +const mode = modeArg ? modeArg.split('=')[1] : 'hybrid'; +const query = args.filter(a => !a.startsWith('--')).join(' '); + +if (!query) { + console.error('Usage: node scripts/search-developers.js "your search query" [--mode=hybrid|vector|text]'); + process.exit(1); +} + +async function getQueryEmbedding(text) { + const url = `${OPENAI_ENDPOINT}/openai/deployments/${EMBEDDING_DEPLOYMENT}/embeddings?api-version=2024-02-01`; + const response = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'api-key': OPENAI_KEY }, + body: JSON.stringify({ input: [text] }) + }); + const data = await response.json(); + return data.data[0].embedding; +} + +async function main() { + const client = new CosmosClient({ endpoint: COSMOS_ENDPOINT, key: COSMOS_KEY }); + const container = client.database(DATABASE_NAME).container(CONTAINER_NAME); + + console.log(`\nšŸ” Searching: "${query}" (mode: ${mode})\n`); + + let results; + + if (mode === 'vector') { + // ā”€ā”€ā”€ Pure Vector Search ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + // Semantic search: finds developers by meaning, not exact words + const embedding = await getQueryEmbedding(query); + + const { resources } = await container.items.query({ + query: ` + SELECT TOP 10 + c.login, c.name, c.location, c.topLanguage, c.score, + c.totalStars, c.followers, c.soReputation, + VectorDistance(c.embedding, @embedding) AS similarityScore + FROM c + WHERE VectorDistance(c.embedding, @embedding) > 0.7 + ORDER BY VectorDistance(c.embedding, @embedding) + `, + parameters: [{ name: '@embedding', value: embedding }] + }).fetchAll(); + results = resources; + + } else if (mode === 'text') { + // ā”€ā”€ā”€ Pure Full-Text Search (BM25) ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + // Keyword search: matches exact terms in indexed fields + const { resources } = await container.items.query({ + query: ` + SELECT TOP 10 + c.login, c.name, c.location, c.topLanguage, c.score, + c.totalStars, c.followers, c.soReputation, + FullTextScore(c.login, [@query]) + + FullTextScore(c.name, [@query]) + + FullTextScore(c.location, [@query]) + + FullTextScore(c.bio, [@query]) + + FullTextScore(c.topLanguage, [@query]) AS textScore + FROM c + WHERE FullTextContains(c.login, @query) + OR FullTextContains(c.name, @query) + OR FullTextContains(c.location, @query) + OR FullTextContains(c.bio, @query) + OR FullTextContains(c.topLanguage, @query) + ORDER BY RANK FullTextScore(c.login, [@query]) + + FullTextScore(c.name, [@query]) + + FullTextScore(c.location, [@query]) + + FullTextScore(c.bio, [@query]) + + FullTextScore(c.topLanguage, [@query]) + `, + parameters: [{ name: '@query', value: query }] + }).fetchAll(); + results = resources; + + } else { + // ā”€ā”€ā”€ Hybrid Search (Vector + Full-Text with RRF) ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + // Best of both: semantic understanding + keyword precision + // Uses Reciprocal Rank Fusion (RRF) to combine rankings + const embedding = await getQueryEmbedding(query); + + const { resources } = await container.items.query({ + query: ` + SELECT TOP 10 + c.login, c.name, c.location, c.topLanguage, c.score, + c.totalStars, c.followers, c.soReputation + FROM c + WHERE FullTextContains(c.login, @query) + OR FullTextContains(c.name, @query) + OR FullTextContains(c.location, @query) + OR FullTextContains(c.bio, @query) + OR FullTextContains(c.topLanguage, @query) + OR VectorDistance(c.embedding, @embedding) > 0.7 + ORDER BY RANK RRF( + FullTextScore(c.login, [@query]) + + FullTextScore(c.name, [@query]) + + FullTextScore(c.location, [@query]) + + FullTextScore(c.bio, [@query]) + + FullTextScore(c.topLanguage, [@query]), + VectorDistance(c.embedding, @embedding) + ) + `, + parameters: [ + { name: '@query', value: query }, + { name: '@embedding', value: embedding } + ] + }).fetchAll(); + results = resources; + } + + // Display results + if (results.length === 0) { + console.log(' No results found.\n'); + return; + } + + console.log(` Found ${results.length} developers:\n`); + console.log(' # Login Score Stars Location Language'); + console.log(' ā”€ ā”€ā”€ā”€ā”€ā”€ ā”€ā”€ā”€ā”€ā”€ ā”€ā”€ā”€ā”€ā”€ ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€'); + results.forEach((dev, i) => { + console.log( + ` ${(i + 1).toString().padEnd(2)} ${(dev.login || '').padEnd(18)} ` + + `${(dev.score || 0).toString().padEnd(6)} ` + + `${formatNum(dev.totalStars || 0).padEnd(8)} ` + + `${(dev.location || 'Unknown').slice(0, 20).padEnd(20)} ` + + `${dev.topLanguage || 'N/A'}` + ); + }); + console.log(''); +} + +function formatNum(n) { + if (n >= 1000) return (n / 1000).toFixed(1) + 'k'; + return n.toString(); +} + +main().catch(err => { console.error(err); process.exit(1); }); diff --git a/scripts/setup-vector-search.js b/scripts/setup-vector-search.js new file mode 100644 index 0000000..6d91c2e --- /dev/null +++ b/scripts/setup-vector-search.js @@ -0,0 +1,118 @@ +/** + * Setup Cosmos DB container with vector search + full-text search for hybrid queries + * + * Usage: node scripts/setup-vector-search.js + * + * This recreates the 'developers' container with: + * - Vector embedding policy (1536-dim for text-embedding-3-small) + * - Vector index (quantizedFlat for cost efficiency) + * - Full-text index on searchable fields (for hybrid search) + */ +import 'dotenv/config'; +import { CosmosClient } from '@azure/cosmos'; + +const COSMOS_ENDPOINT = process.env.COSMOS_ENDPOINT; +const COSMOS_KEY = process.env.COSMOS_KEY; +const DATABASE_NAME = 'devglobe'; +const CONTAINER_NAME = 'developers'; + +async function main() { + const client = new CosmosClient({ endpoint: COSMOS_ENDPOINT, key: COSMOS_KEY }); + const database = client.database(DATABASE_NAME); + + console.log('āš™ļø Setting up vector search container...\n'); + + // Container definition with vector embedding policy + const containerDef = { + id: CONTAINER_NAME, + partitionKey: { paths: ['/location'] }, + indexingPolicy: { + indexingMode: 'consistent', + automatic: true, + includedPaths: [{ path: '/*' }], + excludedPaths: [{ path: '/embedding/*' }], + // Full-text indexes for hybrid search + fullTextIndexes: [ + { path: '/login' }, + { path: '/name' }, + { path: '/location' }, + { path: '/bio' }, + { path: '/topLanguage' } + ], + // Vector index + vectorIndexes: [ + { + path: '/embedding', + type: 'quantizedFlat' // Good for < 100K docs. Use 'diskANN' for larger datasets + } + ] + }, + // Vector embedding policy ā€” defines how vectors are stored + vectorEmbeddingPolicy: { + vectorEmbeddings: [ + { + path: '/embedding', + dataType: 'float32', + dimensions: 1536, // text-embedding-3-small + distanceFunction: 'cosine' + } + ] + }, + // Full-text policy for BM25 text ranking + fullTextPolicy: { + defaultLanguage: 'en-US', + fullTextPaths: [ + { path: '/login', language: 'en-US' }, + { path: '/name', language: 'en-US' }, + { path: '/location', language: 'en-US' }, + { path: '/bio', language: 'en-US' }, + { path: '/topLanguage', language: 'en-US' } + ] + } + }; + + // Delete and recreate container (WARNING: deletes existing data!) + console.log('āš ļø This will delete and recreate the container.'); + console.log(' Make sure you have the pipeline to re-upload data.\n'); + + try { + // Test if vector policy is supported before deleting + const testContainer = { + id: '_vector_test_' + Date.now(), + partitionKey: { paths: ['/id'] }, + vectorEmbeddingPolicy: { + vectorEmbeddings: [{ + path: '/embedding', dataType: 'float32', dimensions: 3, distanceFunction: 'cosine' + }] + } + }; + const { container: testC } = await database.containers.create(testContainer); + await testC.delete(); + console.log(' āœ“ Vector search capability confirmed\n'); + } catch (e) { + if (e.body?.message?.includes('not been enabled')) { + console.error('āŒ Vector search capability is not yet propagated on your account.'); + console.error(' The capability was enabled but needs time to propagate (15-30 min).'); + console.error(' Re-run this script in a few minutes: node scripts/setup-vector-search.js'); + process.exit(1); + } + throw e; + } + + try { + await database.container(CONTAINER_NAME).delete(); + console.log(' Deleted existing container'); + } catch (e) { + if (e.code !== 404) throw e; + } + + const { container } = await database.containers.create(containerDef); + console.log(`āœ… Created container "${CONTAINER_NAME}" with vector + full-text indexes`); + console.log(' Vector: 1536-dim, cosine, quantizedFlat'); + console.log(' Full-text: login, name, location, bio, topLanguage\n'); + console.log('Next steps:'); + console.log(' 1. Run: node scripts/generate-embeddings.js (generate & upload embeddings)'); + console.log(' 2. Query with: node scripts/search-developers.js "your query"'); +} + +main().catch(err => { console.error(err); process.exit(1); }); diff --git a/scripts/vector-policy.json b/scripts/vector-policy.json new file mode 100644 index 0000000..0dbed8a --- /dev/null +++ b/scripts/vector-policy.json @@ -0,0 +1 @@ +[{"path":"/embedding","dataType":"float32","dimensions":1536,"distanceFunction":"cosine"}] diff --git a/server.js b/server.js index 7fe01d3..9a48b3a 100644 --- a/server.js +++ b/server.js @@ -53,6 +53,47 @@ app.get('/api/developers', async (req, res) => { } }); +// Search endpoint ā€” supports text search now, upgrades to hybrid when vector is ready +app.get('/api/search', async (req, res) => { + res.setHeader('Content-Type', 'application/json'); + + const { q, mode = 'text', top = '10' } = req.query; + if (!q) return res.status(400).json({ error: 'Query parameter "q" is required' }); + + try { + const client = new CosmosClient({ endpoint: COSMOS_ENDPOINT, key: COSMOS_KEY }); + const container = client.database(DATABASE).container(CONTAINER); + const limit = Math.min(parseInt(top) || 10, 50); + const searchTerm = q.toLowerCase(); + + const { resources } = await container.items.query({ + query: ` + SELECT TOP @limit + c.id, c.login, c.name, c.avatarUrl, c.location, c.lat, c.lng, + c.topLanguage, c.score, c.totalStars, c.followers, c.soReputation, + c.bio, c.totalCommits, c.scoreDimensions + FROM c + WHERE CONTAINS(LOWER(c.login), @q) + OR CONTAINS(LOWER(c.name), @q) + OR CONTAINS(LOWER(c.location), @q) + OR CONTAINS(LOWER(c.bio), @q) + OR CONTAINS(LOWER(c.topLanguage), @q) + ORDER BY c.score DESC + `, + parameters: [ + { name: '@q', value: searchTerm }, + { name: '@limit', value: limit } + ] + }).fetchAll(); + + console.log(`šŸ” Search "${q}" ā†’ ${resources.length} results`); + res.json({ query: q, mode, count: resources.length, results: resources }); + } catch (err) { + console.error('Search error:', err.message); + res.status(500).json({ error: 'Search failed' }); + } +}); + // Serve static files app.use(express.static(__dirname)); From 70ce88624b680c4107cd95e0de233933135fde2f Mon Sep 17 00:00:00 2001 From: sajeetharan Date: Wed, 29 Jul 2026 17:14:49 +0530 Subject: [PATCH 2/3] fix: working vector + hybrid search with dedicated throughput - Fixed container creation: vector indexes require dedicated throughput (not shared) - Added safety check: validates capability before deleting container - Fixed text search to use CONTAINS instead of FullTextContains (more compatible) - Hybrid search uses client-side RRF to merge vector + text results - Verified: vector search finds semantically similar devs (Evan You for 'JS frontend') --- scripts/search-developers.js | 94 +++++++++++++++++++--------------- scripts/setup-vector-search.js | 2 +- 2 files changed, 55 insertions(+), 41 deletions(-) diff --git a/scripts/search-developers.js b/scripts/search-developers.js index a9a28d0..b7ab3fe 100644 --- a/scripts/search-developers.js +++ b/scripts/search-developers.js @@ -79,61 +79,75 @@ async function main() { query: ` SELECT TOP 10 c.login, c.name, c.location, c.topLanguage, c.score, - c.totalStars, c.followers, c.soReputation, - FullTextScore(c.login, [@query]) + - FullTextScore(c.name, [@query]) + - FullTextScore(c.location, [@query]) + - FullTextScore(c.bio, [@query]) + - FullTextScore(c.topLanguage, [@query]) AS textScore + c.totalStars, c.followers, c.soReputation FROM c - WHERE FullTextContains(c.login, @query) - OR FullTextContains(c.name, @query) - OR FullTextContains(c.location, @query) - OR FullTextContains(c.bio, @query) - OR FullTextContains(c.topLanguage, @query) - ORDER BY RANK FullTextScore(c.login, [@query]) + - FullTextScore(c.name, [@query]) + - FullTextScore(c.location, [@query]) + - FullTextScore(c.bio, [@query]) + - FullTextScore(c.topLanguage, [@query]) + WHERE CONTAINS(LOWER(c.login), @query) + OR CONTAINS(LOWER(c.name), @query) + OR CONTAINS(LOWER(c.location), @query) + OR CONTAINS(LOWER(c.bio), @query) + OR CONTAINS(LOWER(c.topLanguage), @query) + ORDER BY c.score DESC `, - parameters: [{ name: '@query', value: query }] + parameters: [{ name: '@query', value: query.toLowerCase() }] }).fetchAll(); results = resources; } else { - // ā”€ā”€ā”€ Hybrid Search (Vector + Full-Text with RRF) ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ - // Best of both: semantic understanding + keyword precision - // Uses Reciprocal Rank Fusion (RRF) to combine rankings + // ā”€ā”€ā”€ Hybrid Search (Vector + Text) ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + // Combines semantic vector similarity with keyword matching + if (!OPENAI_ENDPOINT || !OPENAI_KEY) { + return console.error('OpenAI credentials required for hybrid search'); + } const embedding = await getQueryEmbedding(query); - const { resources } = await container.items.query({ + // Get vector results + const { resources: vectorResults } = await container.items.query({ + query: ` + SELECT TOP 10 + c.login, c.name, c.location, c.topLanguage, c.score, + c.totalStars, c.followers, c.soReputation, + VectorDistance(c.embedding, @embedding) AS similarity + FROM c + ORDER BY VectorDistance(c.embedding, @embedding) + `, + parameters: [{ name: '@embedding', value: embedding }] + }).fetchAll(); + + // Get text results + const searchLower = query.toLowerCase(); + const { resources: textResults } = await container.items.query({ query: ` SELECT TOP 10 c.login, c.name, c.location, c.topLanguage, c.score, c.totalStars, c.followers, c.soReputation FROM c - WHERE FullTextContains(c.login, @query) - OR FullTextContains(c.name, @query) - OR FullTextContains(c.location, @query) - OR FullTextContains(c.bio, @query) - OR FullTextContains(c.topLanguage, @query) - OR VectorDistance(c.embedding, @embedding) > 0.7 - ORDER BY RANK RRF( - FullTextScore(c.login, [@query]) + - FullTextScore(c.name, [@query]) + - FullTextScore(c.location, [@query]) + - FullTextScore(c.bio, [@query]) + - FullTextScore(c.topLanguage, [@query]), - VectorDistance(c.embedding, @embedding) - ) + WHERE CONTAINS(LOWER(c.login), @q) + OR CONTAINS(LOWER(c.name), @q) + OR CONTAINS(LOWER(c.location), @q) + OR CONTAINS(LOWER(c.bio), @q) + OR CONTAINS(LOWER(c.topLanguage), @q) + ORDER BY c.score DESC `, - parameters: [ - { name: '@query', value: query }, - { name: '@embedding', value: embedding } - ] + parameters: [{ name: '@q', value: searchLower }] }).fetchAll(); - results = resources; + + // Client-side RRF (Reciprocal Rank Fusion) + const rrf = new Map(); + const k = 60; // RRF constant + vectorResults.forEach((r, i) => { + rrf.set(r.login, (rrf.get(r.login) || 0) + 1 / (k + i + 1)); + }); + textResults.forEach((r, i) => { + rrf.set(r.login, (rrf.get(r.login) || 0) + 1 / (k + i + 1)); + }); + + // Merge and sort by RRF score + const allResults = new Map(); + [...vectorResults, ...textResults].forEach(r => allResults.set(r.login, r)); + results = [...rrf.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, 10) + .map(([login]) => allResults.get(login)); } // Display results diff --git a/scripts/setup-vector-search.js b/scripts/setup-vector-search.js index 6d91c2e..210eaa8 100644 --- a/scripts/setup-vector-search.js +++ b/scripts/setup-vector-search.js @@ -106,7 +106,7 @@ async function main() { if (e.code !== 404) throw e; } - const { container } = await database.containers.create(containerDef); + const { container } = await database.containers.create(containerDef, { offerThroughput: 1000 }); console.log(`āœ… Created container "${CONTAINER_NAME}" with vector + full-text indexes`); console.log(' Vector: 1536-dim, cosine, quantizedFlat'); console.log(' Full-text: login, name, location, bio, topLanguage\n'); From 5feefbf78da2180bf19318fcfb4434381b447d2f Mon Sep 17 00:00:00 2001 From: sajeetharan Date: Wed, 29 Jul 2026 17:34:24 +0530 Subject: [PATCH 3/3] feat: implement text, vector, and hybrid search modes with UI updates --- index.html | 5 +++ server.js | 92 ++++++++++++++++++++++++++++++++++++++-------- src/leaderboard.js | 64 +++++++++++++++++++++++++++++++- styles/main.css | 23 +++++++++++- 4 files changed, 167 insertions(+), 17 deletions(-) diff --git a/index.html b/index.html index 03d94e1..da04f3c 100644 --- a/index.html +++ b/index.html @@ -19,6 +19,11 @@

DevGlobe

+
diff --git a/server.js b/server.js index 9a48b3a..982b8a5 100644 --- a/server.js +++ b/server.js @@ -53,7 +53,7 @@ app.get('/api/developers', async (req, res) => { } }); -// Search endpoint ā€” supports text search now, upgrades to hybrid when vector is ready +// Search endpoint ā€” supports text, vector, and hybrid search app.get('/api/search', async (req, res) => { res.setHeader('Content-Type', 'application/json'); @@ -64,11 +64,75 @@ app.get('/api/search', async (req, res) => { const client = new CosmosClient({ endpoint: COSMOS_ENDPOINT, key: COSMOS_KEY }); const container = client.database(DATABASE).container(CONTAINER); const limit = Math.min(parseInt(top) || 10, 50); - const searchTerm = q.toLowerCase(); - const { resources } = await container.items.query({ - query: ` - SELECT TOP @limit + let results; + + if (mode === 'vector' || mode === 'hybrid') { + const OPENAI_ENDPOINT = process.env.AZURE_OPENAI_ENDPOINT; + const OPENAI_KEY = process.env.AZURE_OPENAI_KEY; + const DEPLOYMENT = process.env.EMBEDDING_DEPLOYMENT || 'text-embedding-3-small'; + + if (!OPENAI_ENDPOINT || !OPENAI_KEY) { + return res.status(500).json({ error: 'OpenAI not configured for vector search' }); + } + + // Generate embedding for the query + const embRes = await fetch( + `${OPENAI_ENDPOINT}/openai/deployments/${DEPLOYMENT}/embeddings?api-version=2024-02-01`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'api-key': OPENAI_KEY }, + body: JSON.stringify({ input: [q] }) + } + ); + const embData = await embRes.json(); + const embedding = embData.data[0].embedding; + + // Vector search + const { resources: vectorResults } = await container.items.query({ + query: `SELECT TOP ${limit} + c.id, c.login, c.name, c.avatarUrl, c.location, c.lat, c.lng, + c.topLanguage, c.score, c.totalStars, c.followers, c.soReputation, + c.bio, c.totalCommits, c.scoreDimensions, + VectorDistance(c.embedding, @emb) AS similarity + FROM c ORDER BY VectorDistance(c.embedding, @emb)`, + parameters: [{ name: '@emb', value: embedding }] + }).fetchAll(); + + if (mode === 'vector') { + results = vectorResults; + } else { + // Hybrid: also run text search and merge with RRF + const searchTerm = q.toLowerCase(); + const { resources: textResults } = await container.items.query({ + query: `SELECT TOP ${limit} + c.id, c.login, c.name, c.avatarUrl, c.location, c.lat, c.lng, + c.topLanguage, c.score, c.totalStars, c.followers, c.soReputation, + c.bio, c.totalCommits, c.scoreDimensions + FROM c + WHERE CONTAINS(LOWER(c.login), @q) + OR CONTAINS(LOWER(c.name), @q) + OR CONTAINS(LOWER(c.location), @q) + OR CONTAINS(LOWER(c.bio), @q) + OR CONTAINS(LOWER(c.topLanguage), @q) + ORDER BY c.score DESC`, + parameters: [{ name: '@q', value: searchTerm }] + }).fetchAll(); + + // Client-side RRF + const rrf = new Map(); + const k = 60; + vectorResults.forEach((r, i) => rrf.set(r.login, (rrf.get(r.login) || 0) + 1 / (k + i + 1))); + textResults.forEach((r, i) => rrf.set(r.login, (rrf.get(r.login) || 0) + 1 / (k + i + 1))); + const allMap = new Map(); + [...vectorResults, ...textResults].forEach(r => allMap.set(r.login, r)); + results = [...rrf.entries()].sort((a, b) => b[1] - a[1]).slice(0, limit).map(([login]) => allMap.get(login)); + } + } else { + // Text search + const searchTerm = q.toLowerCase(); + const { resources } = await container.items.query({ + query: `SELECT TOP ${limit} c.id, c.login, c.name, c.avatarUrl, c.location, c.lat, c.lng, c.topLanguage, c.score, c.totalStars, c.followers, c.soReputation, c.bio, c.totalCommits, c.scoreDimensions @@ -78,16 +142,14 @@ app.get('/api/search', async (req, res) => { OR CONTAINS(LOWER(c.location), @q) OR CONTAINS(LOWER(c.bio), @q) OR CONTAINS(LOWER(c.topLanguage), @q) - ORDER BY c.score DESC - `, - parameters: [ - { name: '@q', value: searchTerm }, - { name: '@limit', value: limit } - ] - }).fetchAll(); - - console.log(`šŸ” Search "${q}" ā†’ ${resources.length} results`); - res.json({ query: q, mode, count: resources.length, results: resources }); + ORDER BY c.score DESC`, + parameters: [{ name: '@q', value: searchTerm }] + }).fetchAll(); + results = resources; + } + + console.log(`šŸ” Search "${q}" (${mode}) ā†’ ${results.length} results`); + res.json({ query: q, mode, count: results.length, results }); } catch (err) { console.error('Search error:', err.message); res.status(500).json({ error: 'Search failed' }); diff --git a/src/leaderboard.js b/src/leaderboard.js index 1113032..f7bfc66 100644 --- a/src/leaderboard.js +++ b/src/leaderboard.js @@ -8,9 +8,11 @@ const Leaderboard = (() => { const filterCountry = document.getElementById('filter-country'); const filterLang = document.getElementById('filter-language'); const filterSort = document.getElementById('filter-sort'); + const searchMode = document.getElementById('search-mode'); let allDevelopers = []; let filteredDevelopers = []; + let activeSearchAbort = null; // Virtual scrolling state const ITEM_HEIGHT = 62; @@ -40,7 +42,34 @@ const Leaderboard = (() => { let searchTimer; searchInput.addEventListener('input', () => { clearTimeout(searchTimer); - searchTimer = setTimeout(applyFilters, 200); + searchTimer = setTimeout(() => { + const mode = searchMode.value; + if (mode === 'vector' || mode === 'hybrid') { + apiSearch(); + } else { + applyFilters(); + } + }, 400); + }); + searchInput.addEventListener('keydown', (e) => { + if (e.key === 'Enter') { + clearTimeout(searchTimer); + const mode = searchMode.value; + if (mode === 'vector' || mode === 'hybrid') { + apiSearch(); + } else { + applyFilters(); + } + } + }); + searchMode.addEventListener('change', () => { + if (!searchInput.value.trim()) return; + const mode = searchMode.value; + if (mode === 'vector' || mode === 'hybrid') { + apiSearch(); + } else { + applyFilters(); + } }); filterCountry.addEventListener('change', applyFilters); filterLang.addEventListener('change', applyFilters); @@ -119,6 +148,39 @@ const Leaderboard = (() => { GlobeViz.updateData(filteredDevelopers); } + async function apiSearch() { + const query = searchInput.value.trim(); + if (!query) { applyFilters(); return; } + + if (activeSearchAbort) activeSearchAbort.abort(); + const controller = new AbortController(); + activeSearchAbort = controller; + + const mode = searchMode.value; + searchInput.style.opacity = '0.5'; + + try { + const res = await fetch( + `/api/search?q=${encodeURIComponent(query)}&mode=${mode}&top=20`, + { signal: controller.signal } + ); + const data = await res.json(); + if (controller.signal.aborted) return; + + filteredDevelopers = data.results || []; + // Compute scores using the full dataset's max values + filteredDevelopers = Scoring.scoreAll(filteredDevelopers); + renderedRange = { start: -1, end: -1 }; + listEl.scrollTop = 0; + renderVirtual(); + GlobeViz.updateData(filteredDevelopers); + } catch (e) { + if (e.name !== 'AbortError') console.error('Search failed:', e); + } finally { + if (!controller.signal.aborted) searchInput.style.opacity = '1'; + } + } + function renderVirtual() { const devs = filteredDevelopers; const totalHeight = devs.length * ITEM_HEIGHT; diff --git a/styles/main.css b/styles/main.css index 1bc3387..4a3439c 100644 --- a/styles/main.css +++ b/styles/main.css @@ -71,6 +71,12 @@ body { font-weight: 400; } +.header__search { + display: flex; + gap: 6px; + align-items: center; +} + .header__search input { width: 260px; padding: 8px 14px; @@ -80,13 +86,28 @@ body { color: var(--text-primary); font-size: 13px; outline: none; - transition: border-color 0.2s; + transition: border-color 0.2s, opacity 0.2s; } .header__search input:focus { border-color: var(--accent-blue); } +.header__search select { + padding: 8px 10px; + background: var(--bg-secondary); + border: 1px solid var(--border); + border-radius: var(--radius); + color: var(--text-primary); + font-size: 12px; + cursor: pointer; + outline: none; +} + +.header__search select:focus { + border-color: var(--accent-blue); +} + /* Main layout */ .main { position: relative;