diff --git a/.gitignore b/.gitignore index 32ea24e..3fdbbcc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +typesense-data bun.lockb # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. @@ -37,4 +38,4 @@ yarn-error.log* *.tsbuildinfo next-env.d.ts -.env \ No newline at end of file +.env diff --git a/app/api/orgs/[orgId]/contests/search/route.ts b/app/api/orgs/[orgId]/contests/search/route.ts new file mode 100644 index 0000000..59af05e --- /dev/null +++ b/app/api/orgs/[orgId]/contests/search/route.ts @@ -0,0 +1,25 @@ +import { NextRequest, NextResponse } from "next/server"; +import { searchContests } from "@/lib/typesense/collections/contests"; +import { safeSearch } from "@/lib/typesense/client"; + +export async function GET( + request: NextRequest, + { params }: { params: { orgId: string } }, +) { + const searchParams = request.nextUrl.searchParams; + const query = searchParams.get("q") || ""; + const page = parseInt(searchParams.get("page") || "1", 10); + const per_page = parseInt(searchParams.get("per_page") || "10", 10); + const orgId = parseInt(params.orgId, 10); + + try { + const results = await safeSearch( + () => searchContests(query, orgId, { page, per_page }), + { found: 0, hits: [], page }, + ); + return NextResponse.json(results); + } catch (error) { + console.error("Contests search error:", error); + return NextResponse.json({ error: "Search failed" }, { status: 500 }); + } +} diff --git a/app/api/orgs/[orgId]/groups/search/route.ts b/app/api/orgs/[orgId]/groups/search/route.ts new file mode 100644 index 0000000..173dbdb --- /dev/null +++ b/app/api/orgs/[orgId]/groups/search/route.ts @@ -0,0 +1,25 @@ +import { NextRequest, NextResponse } from "next/server"; +import { searchGroups } from "@/lib/typesense/collections/groups"; +import { safeSearch } from "@/lib/typesense/client"; + +export async function GET( + request: NextRequest, + { params }: { params: { orgId: string } }, +) { + const searchParams = request.nextUrl.searchParams; + const query = searchParams.get("q") || ""; + const page = parseInt(searchParams.get("page") || "1", 10); + const per_page = parseInt(searchParams.get("per_page") || "10", 10); + const orgId = parseInt(params.orgId, 10); + + try { + const results = await safeSearch( + () => searchGroups(query, orgId, { page, per_page }), + { found: 0, hits: [], page }, + ); + return NextResponse.json(results); + } catch (error) { + console.error("Groups search error:", error); + return NextResponse.json({ error: "Search failed" }, { status: 500 }); + } +} diff --git a/app/api/orgs/[orgId]/problems/search/route.ts b/app/api/orgs/[orgId]/problems/search/route.ts new file mode 100644 index 0000000..3da390b --- /dev/null +++ b/app/api/orgs/[orgId]/problems/search/route.ts @@ -0,0 +1,25 @@ +import { NextRequest, NextResponse } from "next/server"; +import { searchProblems } from "@/lib/typesense/collections/problems"; +import { safeSearch } from "@/lib/typesense/client"; + +export async function GET( + request: NextRequest, + { params }: { params: { orgId: string } }, +) { + const searchParams = request.nextUrl.searchParams; + const query = searchParams.get("q") || ""; + const page = parseInt(searchParams.get("page") || "1", 10); + const per_page = parseInt(searchParams.get("per_page") || "10", 10); + const orgId = parseInt(params.orgId, 10); + + try { + const results = await safeSearch( + () => searchProblems(query, orgId, { page, per_page }), + { found: 0, hits: [], page }, + ); + return NextResponse.json(results); + } catch (error) { + console.error("Problems search error:", error); + return NextResponse.json({ error: "Search failed" }, { status: 500 }); + } +} diff --git a/app/api/orgs/[orgId]/submissions/search/route.ts b/app/api/orgs/[orgId]/submissions/search/route.ts new file mode 100644 index 0000000..b6ab075 --- /dev/null +++ b/app/api/orgs/[orgId]/submissions/search/route.ts @@ -0,0 +1,40 @@ +import { NextRequest, NextResponse } from "next/server"; +import { filterSubmissions } from "@/lib/typesense/collections/submissions"; +import { safeSearch } from "@/lib/typesense/client"; + +export async function GET( + request: NextRequest, + { params }: { params: { orgId: string } }, +) { + const searchParams = request.nextUrl.searchParams; + const orgId = parseInt(params.orgId, 10); + + // Extract filter parameters + const options = { + userNameId: searchParams.get("user") || undefined, + contestNameId: searchParams.get("contest") || undefined, + problemNameId: searchParams.get("problem") || undefined, + language: searchParams.get("language") || undefined, + status: searchParams.get("status") || undefined, + startTime: searchParams.get("start") + ? parseInt(searchParams.get("start")!, 10) + : undefined, + endTime: searchParams.get("end") + ? parseInt(searchParams.get("end")!, 10) + : undefined, + page: parseInt(searchParams.get("page") || "1", 10), + per_page: parseInt(searchParams.get("per_page") || "10", 10), + }; + + try { + const query = searchParams.get("q") || ""; + const results = await safeSearch( + () => searchSubmissions(query, orgId, options), + { found: 0, hits: [], page: options.page }, + ); + return NextResponse.json(results); + } catch (error) { + console.error("Submissions search error:", error); + return NextResponse.json({ error: "Search failed" }, { status: 500 }); + } +} diff --git a/app/api/orgs/[orgId]/users/search/route.ts b/app/api/orgs/[orgId]/users/search/route.ts new file mode 100644 index 0000000..cc740c5 --- /dev/null +++ b/app/api/orgs/[orgId]/users/search/route.ts @@ -0,0 +1,25 @@ +import { NextRequest, NextResponse } from "next/server"; +import { searchUsers } from "@/lib/typesense/collections/users"; +import { safeSearch } from "@/lib/typesense/client"; + +export async function GET( + request: NextRequest, + { params }: { params: { orgId: string } }, +) { + const searchParams = request.nextUrl.searchParams; + const query = searchParams.get("q") || ""; + const page = parseInt(searchParams.get("page") || "1", 10); + const per_page = parseInt(searchParams.get("per_page") || "10", 10); + const orgId = parseInt(params.orgId, 10); + + try { + const results = await safeSearch( + () => searchUsers(query, orgId, { page, per_page }), + { found: 0, hits: [], page }, + ); + return NextResponse.json(results); + } catch (error) { + console.error("Users search error:", error); + return NextResponse.json({ error: "Search failed" }, { status: 500 }); + } +} diff --git a/app/api/search/contests/route.ts b/app/api/search/contests/route.ts new file mode 100644 index 0000000..19e6953 --- /dev/null +++ b/app/api/search/contests/route.ts @@ -0,0 +1,20 @@ +import { NextRequest, NextResponse } from "next/server"; +import { searchContests } from "@/lib/typesense/collections/contests"; +import { safeSearch } from "@/lib/typesense/client"; + +export async function GET(request: NextRequest) { + const searchParams = request.nextUrl.searchParams; + const query = searchParams.get("q") || ""; + const page = parseInt(searchParams.get("page") || "1", 10); + const per_page = parseInt(searchParams.get("per_page") || "10", 10); + + try { + const results = await safeSearch( + () => searchContests(query, { page, per_page }), + { found: 0, hits: [], page }, + ); + return NextResponse.json(results); + } catch (error) { + return NextResponse.json({ error: "Search failed" }, { status: 500 }); + } +} diff --git a/app/api/search/orgs/route.ts b/app/api/search/orgs/route.ts new file mode 100644 index 0000000..2755d03 --- /dev/null +++ b/app/api/search/orgs/route.ts @@ -0,0 +1,20 @@ +import { NextRequest, NextResponse } from "next/server"; +import { searchOrgs } from "@/lib/typesense/collections/orgs"; +import { safeSearch } from "@/lib/typesense/client"; + +export async function GET(request: NextRequest) { + const searchParams = request.nextUrl.searchParams; + const query = searchParams.get("q") || ""; + const page = parseInt(searchParams.get("page") || "1", 10); + const per_page = parseInt(searchParams.get("per_page") || "10", 10); + + try { + const results = await safeSearch( + () => searchOrgs(query, { page, per_page }), + { found: 0, hits: [], page }, + ); + return NextResponse.json(results); + } catch (error) { + return NextResponse.json({ error: "Search failed" }, { status: 500 }); + } +} diff --git a/app/api/search/problems/route.ts b/app/api/search/problems/route.ts new file mode 100644 index 0000000..c5330ca --- /dev/null +++ b/app/api/search/problems/route.ts @@ -0,0 +1,20 @@ +import { NextRequest, NextResponse } from "next/server"; +import { searchProblems } from "@/lib/typesense/collections/problems"; +import { safeSearch } from "@/lib/typesense/client"; + +export async function GET(request: NextRequest) { + const searchParams = request.nextUrl.searchParams; + const query = searchParams.get("q") || ""; + const page = parseInt(searchParams.get("page") || "1", 10); + const per_page = parseInt(searchParams.get("per_page") || "10", 10); + + try { + const results = await safeSearch( + () => searchProblems(query, { page, per_page }), + { found: 0, hits: [], page }, + ); + return NextResponse.json(results); + } catch (error) { + return NextResponse.json({ error: "Search failed" }, { status: 500 }); + } +} diff --git a/app/api/search/users/route.ts b/app/api/search/users/route.ts new file mode 100644 index 0000000..7b9daa7 --- /dev/null +++ b/app/api/search/users/route.ts @@ -0,0 +1,20 @@ +import { NextRequest, NextResponse } from "next/server"; +import { searchUsers } from "@/lib/typesense/collections/users"; +import { safeSearch } from "@/lib/typesense/client"; + +export async function GET(request: NextRequest) { + const searchParams = request.nextUrl.searchParams; + const query = searchParams.get("q") || ""; + const page = parseInt(searchParams.get("page") || "1", 10); + const per_page = parseInt(searchParams.get("per_page") || "10", 10); + + try { + const results = await safeSearch( + () => searchUsers(query, { page, per_page }), + { found: 0, hits: [], page }, + ); + return NextResponse.json(results); + } catch (error) { + return NextResponse.json({ error: "Search failed" }, { status: 500 }); + } +} diff --git a/app/api/typesense/health/route.ts b/app/api/typesense/health/route.ts new file mode 100644 index 0000000..0521b6f --- /dev/null +++ b/app/api/typesense/health/route.ts @@ -0,0 +1,15 @@ +import { testTypesenseConnection } from "@/lib/typesense/test-connection"; +import { NextResponse } from "next/server"; + +export async function GET() { + const isHealthy = await testTypesenseConnection(); + + if (!isHealthy) { + return NextResponse.json( + { error: "Typesense connection failed" }, + { status: 500 }, + ); + } + + return NextResponse.json({ status: "healthy" }); +} diff --git a/app/layout.tsx b/app/layout.tsx index 8c4cab4..383cd5c 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -6,6 +6,9 @@ import "@uiw/react-markdown-preview/markdown.css"; import { Toaster } from "@/components/ui/toaster"; import { AuthProvider } from "@/contexts/auth-context"; import { ThemeProvider } from "@/contexts/theme-context"; // Added import statement +import { initializeServices } from "@/lib/init"; + +initializeServices().catch(console.error); const geistSans = localFont({ src: "./fonts/GeistVF.woff", diff --git a/bun.lock b/bun.lock index cdd6554..d0f23f4 100644 --- a/bun.lock +++ b/bun.lock @@ -57,6 +57,7 @@ "jest": "^29.7.0", "jest-environment-jsdom": "^29.7.0", "js-cookie": "^3.0.5", + "lodash.debounce": "^4.0.8", "lucide-react": "^0.446.0", "marked": "^15.0.6", "next": "14.2.13", @@ -78,12 +79,14 @@ "tailwind-merge": "^2.5.5", "tailwindcss-animate": "^1.0.7", "ts-jest": "^29.3.2", + "typesense": "^2.0.3", "zod": "^3.24.3", }, "devDependencies": { "@radix-ui/react-label": "^2.1.0", "@tailwindcss/typography": "^0.5.15", "@types/codemirror": "^5.60.15", + "@types/lodash.debounce": "^4.0.9", "@types/node": "^20", "@types/nodemailer": "^6.4.17", "@types/pg": "^8.11.10", @@ -970,6 +973,10 @@ "@types/koa__router": ["@types/koa__router@12.0.3", "", { "dependencies": { "@types/koa": "*" } }, "sha512-5YUJVv6NwM1z7m6FuYpKfNLTZ932Z6EF6xy2BbtpJSyn13DKNQEkXVffFVSnJHxvwwWh2SAeumpjAYUELqgjyw=="], + "@types/lodash": ["@types/lodash@4.17.16", "", {}, "sha512-HX7Em5NYQAXKW+1T+FiuG27NGwzJfCX3s1GjOa7ujxZa52kjJLOr4FUxT+giF6Tgxv1e+/czV/iTtBw27WTU9g=="], + + "@types/lodash.debounce": ["@types/lodash.debounce@4.0.9", "", { "dependencies": { "@types/lodash": "*" } }, "sha512-Ma5JcgTREwpLRwMM+XwBR7DaWe96nC38uCBDFKZWbNKD+osjVzdpnUSwBcqCptrp16sSOLBAUb50Car5I0TCsQ=="], + "@types/mdast": ["@types/mdast@4.0.4", "", { "dependencies": { "@types/unist": "*" } }, "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA=="], "@types/memcached": ["@types/memcached@2.2.10", "", { "dependencies": { "@types/node": "*" } }, "sha512-AM9smvZN55Gzs2wRrqeMHVP7KE8KWgCJO/XL5yCly2xF6EKa4YlbpK+cLSAH4NG/Ah64HrlegmGqW8kYws7Vxg=="], @@ -2120,6 +2127,8 @@ "lodash.mergewith": ["lodash.mergewith@4.6.2", "", {}, "sha512-GK3g5RPZWTRSeLSpgP8Xhra+pnjBC56q9FZYe1d5RN3TJ35dbkGy3YqBSMbyCrlbi+CM9Z3Jk5yTL7RCsqboyQ=="], + "loglevel": ["loglevel@1.9.2", "", {}, "sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg=="], + "long": ["long@5.3.1", "", {}, "sha512-ka87Jz3gcx/I7Hal94xaN2tZEOPoUOEVftkQqZx2EeQRN7LGdfLlI3FvZ+7WDplm+vK2Urx9ULrvSowtdCieng=="], "longest-streak": ["longest-streak@3.1.0", "", {}, "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g=="], @@ -2834,6 +2843,8 @@ "typescript": ["typescript@5.8.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ=="], + "typesense": ["typesense@2.0.3", "", { "dependencies": { "axios": "^1.7.2", "loglevel": "^1.8.1", "tslib": "^2.6.2" }, "peerDependencies": { "@babel/runtime": "^7.23.2" } }, "sha512-fRJjFdDNZn6qF9XzIk+bB8n8cm0fiAx1SGcpLDfNcsGtp8znITfG+SO+l/qk63GCRXZwJGq7wrMDLFUvblJSHA=="], + "unbox-primitive": ["unbox-primitive@1.1.0", "", { "dependencies": { "call-bound": "^1.0.3", "has-bigints": "^1.0.2", "has-symbols": "^1.1.0", "which-boxed-primitive": "^1.1.1" } }, "sha512-nWJ91DjeOkej/TA8pXQ3myruKpKEYgqvpw9lz4OPHj/NWFNluYrjbz9j01CJ8yKQd2g4jFoOkINCTW2I5LEEyw=="], "undici-types": ["undici-types@6.19.8", "", {}, "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw=="], diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..9cd3189 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,11 @@ +services: + typesense: + image: typesense/typesense:28.0 + ports: + - "8108:8108" + volumes: + - ./typesense-data:/data + environment: + - TYPESENSE_API_KEY=your_api_key_here + - TYPESENSE_DATA_DIR=/data + - TYPESENSE_ENABLE_CORS=true diff --git a/lib/init.ts b/lib/init.ts new file mode 100644 index 0000000..981e388 --- /dev/null +++ b/lib/init.ts @@ -0,0 +1,20 @@ +import { initializeTypesenseCollections } from "./typesense/init"; +import { initializeTypesenseSync } from "./typesense/subscriber"; +import { syncExistingData } from "./typesense/sync-existing"; + +let initialized = false; + +export async function initializeServices() { + if (initialized) return; + + // Initialize Typesense collections + await initializeTypesenseCollections(); + + // Initialize TypesenseSubscriber + initializeTypesenseSync(); + + // Sync existing data + await syncExistingData(); + + initialized = true; +} diff --git a/lib/typesense/client.ts b/lib/typesense/client.ts new file mode 100644 index 0000000..e4610d5 --- /dev/null +++ b/lib/typesense/client.ts @@ -0,0 +1,87 @@ +import Typesense from "typesense"; +import { config } from "dotenv"; + +config({ path: ".env.local" }); +let typesenseClient: Typesense.Client; + +export function getTypesenseClient() { + if (!typesenseClient) { + typesenseClient = new Typesense.Client({ + nodes: [ + { + host: process.env.TYPESENSE_HOST!, + port: parseInt(process.env.TYPESENSE_PORT!, 10), + protocol: process.env.TYPESENSE_PROTOCOL!, + }, + ], + apiKey: process.env.TYPESENSE_API_KEY!, + connectionTimeoutSeconds: 2, + retryIntervalSeconds: 0.1, + numRetries: 3, + }); + } + + return typesenseClient; +} + +// Helper function to handle search errors +export async function safeSearch( + searchFn: () => Promise, + fallback: T, +): Promise { + try { + return await searchFn(); + } catch (error) { + console.error("Typesense search error:", error); + return fallback; + } +} + +// Types for common search parameters +export interface SearchParams { + q: string; + query_by: string; + query_by_weights?: string; + filter_by?: string; + sort_by?: string; + page?: number; + per_page?: number; + facet_by?: string; + max_facet_values?: number; +} + +// Common search result interface +export interface SearchResponse { + found: number; + hits: Array<{ + document: T; + highlights: Array<{ + field: string; + snippet: string; + }>; + }>; + facet_counts?: Array<{ + field_name: string; + counts: Array<{ + count: number; + value: string; + }>; + }>; + page: number; +} + +// Helper function to create search parameters +export function createSearchParams( + query: string, + queryBy: string[], + weights?: number[], + options: Partial = {}, +): SearchParams { + return { + q: query, + query_by: queryBy.join(","), + query_by_weights: weights?.join(","), + per_page: 10, + ...options, + }; +} diff --git a/lib/typesense/collections/contests.ts b/lib/typesense/collections/contests.ts new file mode 100644 index 0000000..95de583 --- /dev/null +++ b/lib/typesense/collections/contests.ts @@ -0,0 +1,89 @@ +import { + createSearchParams, + getTypesenseClient, + SearchParams, + SearchResponse, +} from "../client"; + +export const CONTESTS_SCHEMA = { + name: "contests", + fields: [ + { name: "id", type: "string" }, + { name: "org_id", type: "int32" }, + { name: "name", type: "string" }, + { name: "name_id", type: "string" }, + { name: "description", type: "string" }, + { name: "problems", type: "string" }, // comma-separated problem codes + { name: "problem_count", type: "int32" }, + { name: "start_time", type: "int64" }, + { name: "end_time", type: "int64" }, + ], + default_sorting_field: "start_time", +}; + +export interface ContestDocument { + id: string; + org_id: number; + name: string; + name_id: string; + description: string; + problems: string; + problem_count: number; + start_time: number; + end_time: number; +} + +export function contestToDocument( + contest: any, + orgId: number, +): ContestDocument { + return { + id: contest.id.toString(), + org_id: orgId, + name: contest.name, + name_id: contest.nameId, + description: contest.description, + problems: contest.problems || "", + problem_count: contest.problems ? contest.problems.split(",").length : 0, + start_time: new Date(contest.startTime).getTime(), + end_time: new Date(contest.endTime).getTime(), + }; +} + +export async function searchContests( + query: string, + orgId: number, + options: Partial = {}, +): Promise> { + const client = getTypesenseClient(); + + const searchParameters = createSearchParams( + query, + ["name", "name_id", "description", "problems"], + [3, 2, 1, 1], + { + filter_by: `org_id:=${orgId}`, + sort_by: "_text_match:desc,start_time:desc", + per_page: options.per_page || 10, + page: options.page || 1, + ...options, + }, + ); + + return await client + .collections("contests") + .documents() + .search(searchParameters); +} + +export async function upsertContest(contest: any, orgId: number) { + const client = getTypesenseClient(); + const document = contestToDocument(contest, orgId); + + return await client.collections("contests").documents().upsert(document); +} + +export async function deleteContest(id: string) { + const client = getTypesenseClient(); + return await client.collections("contests").documents(id).delete(); +} diff --git a/lib/typesense/collections/groups.ts b/lib/typesense/collections/groups.ts new file mode 100644 index 0000000..5c5a391 --- /dev/null +++ b/lib/typesense/collections/groups.ts @@ -0,0 +1,86 @@ +import { + createSearchParams, + getTypesenseClient, + SearchParams, + SearchResponse, +} from "../client"; + +export const GROUPS_SCHEMA = { + name: "groups", + fields: [ + { name: "id", type: "string" }, + { name: "org_id", type: "int32" }, + { name: "name", type: "string" }, + { name: "name_id", type: "string" }, + { name: "about", type: "string", optional: true }, + { name: "avatar", type: "string", optional: true }, + { name: "created_at", type: "int64" }, + { name: "users", type: "string" }, // newline separated emails + { name: "users_count", type: "int32" }, + ], + default_sorting_field: "created_at", +}; + +export interface GroupDocument { + id: string; + org_id: number; + name: string; + name_id: string; + about?: string; + avatar?: string; + created_at: number; + users: string; + users_count: number; +} + +export function groupToDocument(group: any, orgId: number): GroupDocument { + return { + id: group.id.toString(), + org_id: orgId, + name: group.name, + name_id: group.nameId, + about: group.about || undefined, + avatar: group.avatar || undefined, + created_at: new Date(group.createdAt).getTime(), + users: group.userEmails?.join("\n") || "", + users_count: group.userEmails?.length || 0, + }; +} + +export async function searchGroups( + query: string, + orgId: number, + options: Partial = {}, +): Promise> { + const client = getTypesenseClient(); + + const searchParameters = createSearchParams( + query, + ["name", "name_id", "about", "users"], + [3, 2, 1, 1], + { + filter_by: `org_id:=${orgId}`, + sort_by: "_text_match:desc,created_at:desc", + per_page: options.per_page || 10, + page: options.page || 1, + ...options, + }, + ); + + return await client + .collections("groups") + .documents() + .search(searchParameters); +} + +export async function upsertGroup(group: any, orgId: number) { + const client = getTypesenseClient(); + const document = groupToDocument(group, orgId); + + return await client.collections("groups").documents().upsert(document); +} + +export async function deleteGroup(id: string) { + const client = getTypesenseClient(); + return await client.collections("groups").documents(id).delete(); +} diff --git a/lib/typesense/collections/index.ts b/lib/typesense/collections/index.ts new file mode 100644 index 0000000..f945d17 --- /dev/null +++ b/lib/typesense/collections/index.ts @@ -0,0 +1,25 @@ +export * from "./problems"; +export * from "./contests"; +export * from "./users"; +export * from "./orgs"; + +import { getTypesenseClient } from "../client"; +import { PROBLEMS_SCHEMA } from "./problems"; +import { CONTESTS_SCHEMA } from "./contests"; +import { USERS_SCHEMA } from "./users"; +import { ORGS_SCHEMA } from "./orgs"; + +export async function initializeCollections() { + const client = getTypesenseClient(); + + const schemas = [PROBLEMS_SCHEMA, CONTESTS_SCHEMA, USERS_SCHEMA, ORGS_SCHEMA]; + + for (const schema of schemas) { + try { + await client.collections(schema.name).delete(); + } catch (error) { + // Collection might not exist, ignore + } + await client.collections().create(schema); + } +} diff --git a/lib/typesense/collections/orgs.ts b/lib/typesense/collections/orgs.ts new file mode 100644 index 0000000..bf19f4c --- /dev/null +++ b/lib/typesense/collections/orgs.ts @@ -0,0 +1,71 @@ +import { + createSearchParams, + getTypesenseClient, + SearchParams, + SearchResponse, +} from "../client"; +import { SelectOrg } from "@/db/schema"; + +export const ORGS_SCHEMA = { + name: "orgs", + fields: [ + { name: "id", type: "string" }, + { name: "name_id", type: "string" }, + { name: "name", type: "string" }, + { name: "about", type: "string", optional: true }, + { name: "avatar", type: "string", optional: true }, + { name: "created_at", type: "int64" }, + ], + default_sorting_field: "created_at", +}; + +export interface OrgDocument { + id: string; + name_id: string; + name: string; + about?: string; + avatar?: string; + created_at: number; +} + +export function orgToDocument(org: SelectOrg): OrgDocument { + return { + id: org.id.toString(), + name_id: org.nameId, + name: org.name, + about: org.about || undefined, + avatar: org.avatar || undefined, + created_at: new Date(org.createdAt).getTime(), + }; +} + +export async function searchOrgs( + query: string, + options: Partial = {}, +): Promise> { + const client = getTypesenseClient(); + + const searchParameters = createSearchParams( + query, + ["name", "name_id", "about"], + [3, 2, 1], + { + per_page: options.per_page || 10, + page: options.page || 1, + ...options, + }, + ); + + return await client.collections("orgs").documents().search(searchParameters); +} + +export async function upsertOrg(org: SelectOrg) { + const client = getTypesenseClient(); + const document = orgToDocument(org); + return await client.collections("orgs").documents().upsert(document); +} + +export async function deleteOrg(id: number) { + const client = getTypesenseClient(); + return await client.collections("orgs").documents(id.toString()).delete(); +} diff --git a/lib/typesense/collections/problems.ts b/lib/typesense/collections/problems.ts new file mode 100644 index 0000000..4a1c458 --- /dev/null +++ b/lib/typesense/collections/problems.ts @@ -0,0 +1,83 @@ +import { + createSearchParams, + getTypesenseClient, + SearchParams, + SearchResponse, +} from "../client"; + +export const PROBLEMS_SCHEMA = { + name: "problems", + fields: [ + { name: "id", type: "string" }, + { name: "org_id", type: "int32" }, + { name: "code", type: "string" }, + { name: "title", type: "string" }, + { name: "description", type: "string", optional: true }, + { name: "allowed_languages", type: "string[]" }, + { name: "created_at", type: "int64" }, + ], + default_sorting_field: "created_at", +}; + +export interface ProblemDocument { + id: string; + org_id: number; + code: string; + title: string; + description?: string; + allowed_languages: string[]; + created_at: number; +} + +export function problemToDocument( + problem: any, + orgId: number, +): ProblemDocument { + return { + id: problem.id.toString(), + org_id: orgId, + code: problem.code, + title: problem.title, + description: problem.description, + allowed_languages: problem.allowedLanguages, + created_at: new Date(problem.createdAt).getTime(), + }; +} + +export async function searchProblems( + query: string, + orgId: number, + options: Partial = {}, +): Promise> { + const client = getTypesenseClient(); + + const searchParameters = createSearchParams( + query, + ["title", "code", "description"], + [3, 2, 1], + { + filter_by: `org_id:=${orgId}`, + sort_by: "_text_match:desc,created_at:desc", + per_page: options.per_page || 10, + page: options.page || 1, + ...options, + }, + ); + + return await client + .collections("problems") + .documents() + .search(searchParameters); +} + +export async function upsertProblem(problem: any, orgId: number) { + const client = getTypesenseClient(); + const document = problemToDocument(problem, orgId); + + return await client.collections("problems").documents().upsert(document); +} + +export async function deleteProblem(id: string) { + const client = getTypesenseClient(); + return await client.collections("problems").documents(id).delete(); +} diff --git a/lib/typesense/collections/submissions.ts b/lib/typesense/collections/submissions.ts new file mode 100644 index 0000000..99bee90 --- /dev/null +++ b/lib/typesense/collections/submissions.ts @@ -0,0 +1,141 @@ +import { + createSearchParams, + getTypesenseClient, + SearchParams, + SearchResponse, +} from "../client"; + +export const SUBMISSIONS_SCHEMA = { + name: "submissions", + fields: [ + { name: "id", type: "string" }, + { name: "org_id", type: "int32" }, + // User details for searching + { name: "user_name_id", type: "string" }, + { name: "user_name", type: "string" }, + // Contest details for searching + { name: "contest_name_id", type: "string" }, + { name: "contest_name", type: "string" }, + // Problem details for searching + { name: "problem_title", type: "string" }, + { name: "problem_code", type: "string" }, + // Submission details + { name: "language", type: "string" }, + { name: "status", type: "string" }, + { name: "submitted_at", type: "int64" }, + { name: "execution_time", type: "int32" }, + { name: "memory_usage", type: "int32" }, + ], + default_sorting_field: "submitted_at", +}; + +export interface SubmissionDocument { + id: string; + org_id: number; + // User details + user_name_id: string; + user_name: string; + // Contest details + contest_name_id: string; + contest_name: string; + // Problem details + problem_title: string; + problem_code: string; + // Submission details + language: string; + status: string; + submitted_at: number; + execution_time: number; + memory_usage: number; +} + +export function submissionToDocument( + submission: any, + orgId: number, +): SubmissionDocument { + return { + id: submission.id.toString(), + org_id: orgId, + // User details + user_name_id: submission.user.nameId, + user_name: submission.user.name, + // Contest details + contest_name_id: submission.contest.nameId, + contest_name: submission.contest.name, + // Problem details + problem_title: submission.problem.title, + problem_code: submission.problem.id, + // Submission details + language: submission.language, + status: submission.status, + submitted_at: new Date(submission.submittedAt).getTime(), + execution_time: submission.executionTime, + memory_usage: submission.memoryUsage, + }; +} + +export async function searchSubmissions( + query: string, + orgId: number, + options: { + userNameId?: string; + contestNameId?: string; + problemNameId?: string; + language?: string; + status?: string; + startTime?: number; + endTime?: number; + page?: number; + per_page?: number; + } = {}, +): Promise> { + const client = getTypesenseClient(); + + let filterBy = `org_id:=${orgId}`; + if (options.userNameId) filterBy += ` && user_name_id:=${options.userNameId}`; + if (options.contestNameId) + filterBy += ` && contest_name_id:=${options.contestNameId}`; + if (options.problemNameId) + filterBy += ` && contest_problem_name_id:=${options.problemNameId}`; + if (options.language) filterBy += ` && language:=${options.language}`; + if (options.status) filterBy += ` && status:=${options.status}`; + if (options.startTime) filterBy += ` && submitted_at:>=${options.startTime}`; + if (options.endTime) filterBy += ` && submitted_at:<=${options.endTime}`; + + const searchParameters = createSearchParams( + query, + [ + "user_name", + "user_name_id", + "contest_name", + "problem_title", + "problem_code", + ], + [3, 2, 2, 2, 1], + { + filter_by: filterBy, + sort_by: query + ? "_text_match:desc,submitted_at:desc" + : "submitted_at:desc", + per_page: options.per_page || 10, + page: options.page || 1, + }, + ); + + return await client + .collections("submissions") + .documents() + .search(searchParameters); +} + +export async function upsertSubmission(submission: any, orgId: number) { + const client = getTypesenseClient(); + const document = submissionToDocument(submission, orgId); + + return await client.collections("submissions").documents().upsert(document); +} + +export async function deleteSubmission(id: string) { + const client = getTypesenseClient(); + return await client.collections("submissions").documents(id).delete(); +} diff --git a/lib/typesense/collections/users.ts b/lib/typesense/collections/users.ts new file mode 100644 index 0000000..cdf135e --- /dev/null +++ b/lib/typesense/collections/users.ts @@ -0,0 +1,84 @@ +import { + createSearchParams, + getTypesenseClient, + SearchParams, + SearchResponse, +} from "../client"; +import { SelectUser } from "@/db/schema"; + +export const USERS_SCHEMA = { + name: "users", + fields: [ + { name: "id", type: "string" }, + { name: "org_id", type: "int32" }, + { name: "name", type: "string" }, + { name: "name_id", type: "string" }, + { name: "email", type: "string" }, + { name: "about", type: "string", optional: true }, + { name: "avatar", type: "string", optional: true }, + { name: "role", type: "string" }, + { name: "joined_at", type: "int64" }, + ], + default_sorting_field: "joined_at", +}; + +export interface UserDocument { + id: string; + org_id: number; + name: string; + name_id: string; + email: string; + about?: string; + avatar?: string; + role: "owner" | "organizer" | "member"; + joined_at: number; +} + +export function userToDocument(user: any, orgId: number): UserDocument { + return { + id: user.id.toString(), + org_id: orgId, + name: user.name, + name_id: user.nameId, + email: user.email, + about: user.about || undefined, + avatar: user.avatar || undefined, + role: user.role, + joined_at: new Date(user.joinedAt).getTime(), + }; +} + +export async function searchUsers( + query: string, + orgId: number, + options: Partial = {}, +): Promise> { + const client = getTypesenseClient(); + + const searchParameters = createSearchParams( + query, + ["name", "name_id", "about"], + [3, 2, 1], + { + filter_by: `org_id:=${orgId}`, + sort_by: "_text_match:desc,joined_at:desc", + per_page: options.per_page || 10, + page: options.page || 1, + ...options, + }, + ); + + return await client.collections("users").documents().search(searchParameters); +} + +export async function upsertUser(user: any, orgId: number) { + const client = getTypesenseClient(); + const document = userToDocument(user, orgId); + + return await client.collections("users").documents().upsert(document); +} + +export async function deleteUser(id: string) { + const client = getTypesenseClient(); + return await client.collections("users").documents(id).delete(); +} diff --git a/lib/typesense/init.ts b/lib/typesense/init.ts new file mode 100644 index 0000000..64dda83 --- /dev/null +++ b/lib/typesense/init.ts @@ -0,0 +1,24 @@ +import { getTypesenseClient } from "./client"; +import { PROBLEMS_SCHEMA } from "./collections/problems"; +import { CONTESTS_SCHEMA } from "./collections/contests"; +import { USERS_SCHEMA } from "./collections/users"; +import { ORGS_SCHEMA } from "./collections/orgs"; + +export async function initializeTypesenseCollections() { + const client = getTypesenseClient(); + const schemas = [PROBLEMS_SCHEMA, CONTESTS_SCHEMA, USERS_SCHEMA, ORGS_SCHEMA]; + + for (const schema of schemas) { + try { + // Check if collection exists + const exists = await client.collections(schema.name).exists(); + + if (!exists) { + await client.collections().create(schema); + console.log(`Created collection: ${schema.name}`); + } + } catch (error) { + console.error(`Error initializing collection ${schema.name}:`, error); + } + } +} diff --git a/lib/typesense/queue.ts b/lib/typesense/queue.ts new file mode 100644 index 0000000..d165f0b --- /dev/null +++ b/lib/typesense/queue.ts @@ -0,0 +1,54 @@ +import { getRedis } from "@/db/redis"; + +type SyncData = any; + +interface SyncOperation { + operation: "create" | "update" | "delete"; + entity: "problem" | "contest" | "user" | "org"; + data: SyncData; + id: number; + timestamp: number; +} + +export interface BatchedOperations { + problems: { + create: SyncData[]; + update: SyncData[]; + delete: number[]; + }; + contests: { + create: SyncData[]; + update: SyncData[]; + delete: number[]; + }; + users: { + create: SyncData[]; + update: SyncData[]; + delete: number[]; + }; + orgs: { + create: SyncData[]; + update: SyncData[]; + delete: number[]; + }; +} + +export const BATCH_WINDOW = 5000; // 5 seconds +export const MAX_BATCH_SIZE = 100; + +export const publishSync = async ( + operation: "create" | "update" | "delete", + entity: "problem" | "contest" | "user" | "org", + data: SyncData, +) => { + const redis = getRedis(); + const message: SyncOperation = { + operation, + entity, + data, + id: data.id, + timestamp: Date.now(), + }; + + await redis.publish("typesense:sync", JSON.stringify(message)); +}; diff --git a/lib/typesense/subscriber.ts b/lib/typesense/subscriber.ts new file mode 100644 index 0000000..4de0ef9 --- /dev/null +++ b/lib/typesense/subscriber.ts @@ -0,0 +1,106 @@ +import { Redis } from "ioredis"; +import { getRedis } from "@/db/redis"; +import { + syncProblemCreate, + syncProblemUpdate, + syncProblemDelete, + syncContestCreate, + syncContestUpdate, + syncContestDelete, + syncUserCreate, + syncUserUpdate, + syncUserDelete, + syncOrgCreate, + syncOrgUpdate, + syncOrgDelete, +} from "./sync"; +import { type BatchedOperations, MAX_BATCH_SIZE, BATCH_WINDOW } from "./queue"; + +class TypesenseSubscriber { + private batch: BatchedOperations = { + problems: { create: [], update: [], delete: [] }, + contests: { create: [], update: [], delete: [] }, + users: { create: [], update: [], delete: [] }, + orgs: { create: [], update: [], delete: [] }, + }; + + private batchTimeout: NodeJS.Timeout | null = null; + + constructor() { + const redis = new Redis(process.env.REDIS_URL || "redis://localhost:6379"); + + redis.subscribe("typesense:sync", (err) => { + if (err) console.error("Redis subscription error:", err); + }); + + redis.on("message", (channel, message) => { + if (channel === "typesense:sync") { + this.handleMessage(JSON.parse(message)); + } + }); + } + + private handleMessage(message: SyncOperation) { + const { entity, operation, data, id } = message; + + // Add to appropriate batch + this.batch[`${entity}s`][operation].push( + operation === "delete" ? id : data, + ); + + // Reset batch timeout + if (this.batchTimeout) clearTimeout(this.batchTimeout); + + // Process batch if max size reached or after window + if (this.getBatchSize() >= MAX_BATCH_SIZE) { + this.processBatch(); + } else { + this.batchTimeout = setTimeout(() => this.processBatch(), BATCH_WINDOW); + } + } + + private getBatchSize(): number { + return Object.values(this.batch).reduce( + (total, entityOps) => + total + + Object.values(entityOps).reduce((sum, ops) => sum + ops.length, 0), + 0, + ); + } + + private async processBatch() { + const currentBatch = this.batch; + this.batch = { + problems: { create: [], update: [], delete: [] }, + contests: { create: [], update: [], delete: [] }, + users: { create: [], update: [], delete: [] }, + orgs: { create: [], update: [], delete: [] }, + }; + + try { + await Promise.all([ + // Problems + ...currentBatch.problems.create.map(syncProblemCreate), + ...currentBatch.problems.update.map(syncProblemUpdate), + ...currentBatch.problems.delete.map(syncProblemDelete), + // Contests + ...currentBatch.contests.create.map(syncContestCreate), + ...currentBatch.contests.update.map(syncContestUpdate), + ...currentBatch.contests.delete.map(syncContestDelete), + // Users + ...currentBatch.users.create.map(syncUserCreate), + ...currentBatch.users.update.map(syncUserUpdate), + ...currentBatch.users.delete.map(syncUserDelete), + // Organizations + ...currentBatch.orgs.create.map(syncOrgCreate), + ...currentBatch.orgs.update.map(syncOrgUpdate), + ...currentBatch.orgs.delete.map(syncOrgDelete), + ]); + } catch (error) { + console.error("Batch processing error:", error); + // Could add retry logic or error reporting here + } + } +} + +export const initializeTypesenseSync = () => new TypesenseSubscriber(); diff --git a/lib/typesense/sync-existing.ts b/lib/typesense/sync-existing.ts new file mode 100644 index 0000000..ccf7ae8 --- /dev/null +++ b/lib/typesense/sync-existing.ts @@ -0,0 +1,49 @@ +import { db } from "@/db/drizzle"; +import { problems, contests, users, orgs } from "@/db/schema"; +import { publishSync } from "./queue"; +import { getTypesenseClient } from "./client"; + +export async function syncExistingData() { + const client = getTypesenseClient(); + + try { + // Clear existing collections + const collections = ["problems", "contests", "users", "orgs"]; + for (const collection of collections) { + try { + await client + .collections(collection) + .documents() + .delete({ filter_by: "" }); + console.log(`Cleared collection: ${collection}`); + } catch (error) { + console.error(`Error clearing collection ${collection}:`, error); + } + } + + // Sync all existing data + const existingProblems = await db.select().from(problems); + for (const problem of existingProblems) { + await publishSync("create", "problem", problem); + } + + const existingContests = await db.select().from(contests); + for (const contest of existingContests) { + await publishSync("create", "contest", contest); + } + + const existingUsers = await db.select().from(users); + for (const user of existingUsers) { + await publishSync("create", "user", user); + } + + const existingOrgs = await db.select().from(orgs); + for (const org of existingOrgs) { + await publishSync("create", "org", org); + } + + console.log("Existing data sync initiated"); + } catch (error) { + console.error("Error syncing existing data:", error); + } +} diff --git a/lib/typesense/sync.ts b/lib/typesense/sync.ts new file mode 100644 index 0000000..ecd1f55 --- /dev/null +++ b/lib/typesense/sync.ts @@ -0,0 +1,110 @@ +import { + SelectProblem, + SelectContest, + SelectUser, + SelectOrg, +} from "@/db/schema"; +import { upsertProblem, deleteProblem } from "./collections/problems"; +import { upsertContest, deleteContest } from "./collections/contests"; +import { upsertUser, deleteUser } from "./collections/users"; +import { upsertOrg, deleteOrg } from "./collections/orgs"; + +// Problem sync hooks +export async function syncProblemCreate(problem: SelectProblem) { + try { + await upsertProblem(problem); + } catch (error) { + console.error("Failed to sync problem creation:", error); + } +} + +export async function syncProblemUpdate(problem: SelectProblem) { + try { + await upsertProblem(problem); + } catch (error) { + console.error("Failed to sync problem update:", error); + } +} + +export async function syncProblemDelete(id: number) { + try { + await deleteProblem(id); + } catch (error) { + console.error("Failed to sync problem deletion:", error); + } +} + +// Contest sync hooks +export async function syncContestCreate(contest: SelectContest) { + try { + await upsertContest(contest); + } catch (error) { + console.error("Failed to sync contest creation:", error); + } +} + +export async function syncContestUpdate(contest: SelectContest) { + try { + await upsertContest(contest); + } catch (error) { + console.error("Failed to sync contest update:", error); + } +} + +export async function syncContestDelete(id: number) { + try { + await deleteContest(id); + } catch (error) { + console.error("Failed to sync contest deletion:", error); + } +} + +// User sync hooks +export async function syncUserCreate(user: SelectUser) { + try { + await upsertUser(user); + } catch (error) { + console.error("Failed to sync user creation:", error); + } +} + +export async function syncUserUpdate(user: SelectUser) { + try { + await upsertUser(user); + } catch (error) { + console.error("Failed to sync user update:", error); + } +} + +export async function syncUserDelete(id: number) { + try { + await deleteUser(id); + } catch (error) { + console.error("Failed to sync user deletion:", error); + } +} + +// Organization sync hooks +export async function syncOrgCreate(org: SelectOrg) { + try { + await upsertOrg(org); + } catch (error) { + console.error("Failed to sync organization creation:", error); + } +} + +export async function syncOrgUpdate(org: SelectOrg) { + try { + await upsertOrg(org); + } catch (error) { + console.error("Failed to sync organization update:", error); + } +} + +export async function syncOrgDelete(id: number) { + try { + await deleteOrg(id); + } catch (error) { + console.error("Failed to sync organization deletion:", error); + } +} diff --git a/lib/typesense/test-connection.ts b/lib/typesense/test-connection.ts new file mode 100644 index 0000000..dea4e31 --- /dev/null +++ b/lib/typesense/test-connection.ts @@ -0,0 +1,13 @@ +import { getTypesenseClient } from "./client"; + +export async function testTypesenseConnection() { + try { + const client = getTypesenseClient(); + const health = await client.health.retrieve(); + console.log("Typesense connection successful:", health); + return true; + } catch (error) { + console.error("Typesense connection failed:", error); + return false; + } +} diff --git a/package.json b/package.json index 214e661..6a1839f 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,7 @@ "jest": "^29.7.0", "jest-environment-jsdom": "^29.7.0", "js-cookie": "^3.0.5", + "lodash.debounce": "^4.0.8", "lucide-react": "^0.446.0", "marked": "^15.0.6", "next": "14.2.13", @@ -97,12 +98,14 @@ "tailwind-merge": "^2.5.5", "tailwindcss-animate": "^1.0.7", "ts-jest": "^29.3.2", + "typesense": "^2.0.3", "zod": "^3.24.3" }, "devDependencies": { "@radix-ui/react-label": "^2.1.0", "@tailwindcss/typography": "^0.5.15", "@types/codemirror": "^5.60.15", + "@types/lodash.debounce": "^4.0.9", "@types/node": "^20", "@types/nodemailer": "^6.4.17", "@types/pg": "^8.11.10",