From b90019e1c0a14919be0720674362c717bd6d87f0 Mon Sep 17 00:00:00 2001 From: kadami <86646883+kadamidev@users.noreply.github.com> Date: Tue, 19 Aug 2025 05:47:35 +1000 Subject: [PATCH 01/10] migration: queryingData field + GIN index on externalApi table --- .../migration.sql | 2 ++ .../migration.sql | 6 ++++++ .../migration.sql | 2 ++ desci-server/prisma/schema.prisma | 13 +++++++------ 4 files changed, 17 insertions(+), 6 deletions(-) create mode 100644 desci-server/prisma/migrations/20250818192731_add_querying_data_external_api_table/migration.sql create mode 100644 desci-server/prisma/migrations/20250818192759_add_gin_index_external_api_querying_data/migration.sql create mode 100644 desci-server/prisma/migrations/20250818194118_gin_index_ext_api_querying_data/migration.sql diff --git a/desci-server/prisma/migrations/20250818192731_add_querying_data_external_api_table/migration.sql b/desci-server/prisma/migrations/20250818192731_add_querying_data_external_api_table/migration.sql new file mode 100644 index 000000000..702ea6be7 --- /dev/null +++ b/desci-server/prisma/migrations/20250818192731_add_querying_data_external_api_table/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "ExternalApiUsage" ADD COLUMN "queryingData" JSONB; diff --git a/desci-server/prisma/migrations/20250818192759_add_gin_index_external_api_querying_data/migration.sql b/desci-server/prisma/migrations/20250818192759_add_gin_index_external_api_querying_data/migration.sql new file mode 100644 index 000000000..9ecece804 --- /dev/null +++ b/desci-server/prisma/migrations/20250818192759_add_gin_index_external_api_querying_data/migration.sql @@ -0,0 +1,6 @@ +-- CreateIndex: Add GIN index for ExternalApiUsage.queryingData field +-- This allows efficient JSON containment queries on the queryingData field + +CREATE INDEX CONCURRENTLY IF NOT EXISTS "idx_external_api_usage_querying_data_gin" +ON "ExternalApiUsage" USING GIN ("queryingData"); + diff --git a/desci-server/prisma/migrations/20250818194118_gin_index_ext_api_querying_data/migration.sql b/desci-server/prisma/migrations/20250818194118_gin_index_ext_api_querying_data/migration.sql new file mode 100644 index 000000000..2c8d514aa --- /dev/null +++ b/desci-server/prisma/migrations/20250818194118_gin_index_ext_api_querying_data/migration.sql @@ -0,0 +1,2 @@ +-- DropIndex +DROP INDEX "idx_external_api_usage_querying_data_gin"; diff --git a/desci-server/prisma/schema.prisma b/desci-server/prisma/schema.prisma index d803afb0b..fbb690b13 100755 --- a/desci-server/prisma/schema.prisma +++ b/desci-server/prisma/schema.prisma @@ -1435,12 +1435,13 @@ model JournalFormResponse { } model ExternalApiUsage { - id Int @id @default(autoincrement()) - userId Int - apiType ExternalApi - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - data Json? + id Int @id @default(autoincrement()) + userId Int + apiType ExternalApi + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + data Json? // Can dump additional metadata here + queryingData Json? // GIN indexed, important data thats used for querying goes here, avoid dumping random metadata. // Relationships user User @relation(fields: [userId], references: [id]) From b55537eface78b1aa65804f87d586809048058fe Mon Sep 17 00:00:00 2001 From: kadami <86646883+kadamidev@users.noreply.github.com> Date: Tue, 19 Aug 2025 07:16:13 +1000 Subject: [PATCH 02/10] updates for file hash flow changes in ref finder --- .../RefereeRecommender/issuePresignedUrl.ts | 46 +++++++++++++++++-- .../RefereeRecommenderSqsHandler.ts | 23 ++++++++-- .../src/services/sqs/SqsMessageTypes.ts | 1 + 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/desci-server/src/controllers/externalApi/RefereeRecommender/issuePresignedUrl.ts b/desci-server/src/controllers/externalApi/RefereeRecommender/issuePresignedUrl.ts index 3df2ab314..06eb395fb 100644 --- a/desci-server/src/controllers/externalApi/RefereeRecommender/issuePresignedUrl.ts +++ b/desci-server/src/controllers/externalApi/RefereeRecommender/issuePresignedUrl.ts @@ -5,11 +5,14 @@ import { sendSuccess, sendError } from '../../../core/api.js'; import { AuthenticatedRequest } from '../../../core/types.js'; import { logger as parentLogger } from '../../../logger.js'; import { RefereeRecommenderService } from '../../../services/externalApi/RefereeRecommenderService.js'; +import { prisma } from '../../../client.js'; +import { ExternalApi } from '@prisma/client'; const logger = parentLogger.child({ module: 'RefereeRecommender::PresignedUrlController' }); const generatePresignedUrlSchema = z.object({ body: z.object({ + fileHash: z.string().min(1, 'fileHash is required'), fileName: z.string().min(1, 'fileName is required'), }), }); @@ -24,17 +27,52 @@ export const generatePresignedUrl = async (req: AuthenticatedRequest, res: Respo } const { body } = generatePresignedUrlSchema.parse(req); - const { fileName } = body; + const { fileHash, fileName } = body; logger.info( { userId: user.id, + fileHash, fileName, }, - 'Generating presigned URL for referee recommender', + 'Checking cache and potentially generating presigned URL for referee recommender', ); - // Validate file extension (assuming PDFs for now) + // First, check if we have cached results for this fileHash + const cachedResult = await prisma.externalApiUsage.findFirst({ + where: { + userId: user.id, + apiType: ExternalApi.REFEREE_FINDER, + queryingData: { + path: ['fileHash'], + equals: fileHash, + }, + }, + orderBy: { + createdAt: 'desc', // Get the most recent result + }, + }); + + if (cachedResult) { + logger.info( + { + userId: user.id, + fileHash, + cachedResultId: cachedResult.id, + }, + 'Found cached results for fileHash', + ); + + return sendSuccess(res, { + cached: true, + message: 'Results for this file are already available', + resultId: cachedResult.id, + createdAt: cachedResult.createdAt, + }); + } + + // No cached results found, proceed with generating presigned URL + // Validate file extension if (!fileName.toLowerCase().endsWith('.pdf')) { return sendError(res, 'Only PDF files are supported', 400); } @@ -60,8 +98,10 @@ export const generatePresignedUrl = async (req: AuthenticatedRequest, res: Respo ); return sendSuccess(res, { + cached: false, presignedUrl, fileName: generatedFileName, + fileHash, expiresIn: 3600, // 1 hour }); } catch (error) { diff --git a/desci-server/src/services/externalApi/RefereeRecommenderSqsHandler.ts b/desci-server/src/services/externalApi/RefereeRecommenderSqsHandler.ts index 07556aacb..4bf9f0604 100644 --- a/desci-server/src/services/externalApi/RefereeRecommenderSqsHandler.ts +++ b/desci-server/src/services/externalApi/RefereeRecommenderSqsHandler.ts @@ -7,12 +7,18 @@ import { sqsService } from '../sqs/SqsService.js'; import { emitWebsocketEvent, WebSocketEventType } from '../websocketService.js'; import { RefereeRecommenderService, SESSION_TTL_SECONDS } from './RefereeRecommenderService.js'; +import { JsonValue } from 'aws-sdk/clients/appmesh.js'; const logger = parentLogger.child({ module: 'RefereeRecommender::SqsHandler' }); +interface RefereeRecommenderQueryingData { + fileName: string; + eventType: string; + fileHash?: string; +} + interface RefereeRecommenderUsageData { eventType: string; - fileName: string; originalFileName: string; timestamp: string; sessionCreatedAt: number; @@ -198,7 +204,7 @@ export class RefereeRecommenderSqsHandler { where: { userId: session.userId, apiType: ExternalApi.REFEREE_FINDER, - data: { + queryingData: { path: ['fileName'], equals: eventData.file_name, }, @@ -240,9 +246,15 @@ export class RefereeRecommenderSqsHandler { } // Create new record for this session - const usageData: RefereeRecommenderUsageData = { - eventType: eventData.eventType, + // Separate queryable data (GIN indexed) from metadata + const queryingData: RefereeRecommenderQueryingData = { fileName: eventData.file_name, + eventType: eventData.eventType, + ...(eventData.file_hash && { fileHash: eventData.file_hash }), + }; + + const metadataData: RefereeRecommenderUsageData = { + eventType: eventData.eventType, originalFileName: session.originalFileName, timestamp: new Date().toISOString(), sessionCreatedAt: session.createdAt, @@ -253,7 +265,8 @@ export class RefereeRecommenderSqsHandler { data: { userId: session.userId, apiType: ExternalApi.REFEREE_FINDER, - data: usageData, + queryingData: queryingData as unknown as JsonValue, // Fast queries on this field + data: metadataData, // Full metadata storage }, }); diff --git a/desci-server/src/services/sqs/SqsMessageTypes.ts b/desci-server/src/services/sqs/SqsMessageTypes.ts index ef07f21cb..f4dc0f825 100644 --- a/desci-server/src/services/sqs/SqsMessageTypes.ts +++ b/desci-server/src/services/sqs/SqsMessageTypes.ts @@ -28,5 +28,6 @@ export interface ExternalApiSqsMessage extends BaseSqsMessage { messageType: SqsMessageType.EXTERNAL_REFEREE_RECOMMENDER_API; eventType: 'PROCESSING_STARTED' | 'PROCESSING_COMPLETED' | 'PROCESSING_FAILED' | 'PROCESSING_PROGRESS'; file_name: string; // e.g., "referee_rec_v0.1.3_{original_file.pdf}" // Move to hashes later for security. + file_hash?: string; // SHA-256 hash of the file content data?: any; } From 5789e621a3f5ce15b36672d55f87e894abf20714 Mon Sep 17 00:00:00 2001 From: kadami <86646883+kadamidev@users.noreply.github.com> Date: Wed, 20 Aug 2025 05:17:17 +1000 Subject: [PATCH 03/10] deployment var updates, update example env --- .env.example | 2 +- desci-server/kubernetes/deployment_dev.yaml | 2 +- desci-server/kubernetes/deployment_prod.yaml | 2 +- desci-server/kubernetes/deployment_staging.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 68224b25b..4713efdb8 100755 --- a/.env.example +++ b/.env.example @@ -203,7 +203,7 @@ MIXPANEL_TOKEN= # Referee Finder ML_REFEREE_GET_PRESIGNED_URL= -ML_REFEREE_TRIGGER_CID= +ML_REFEREE_TRIGGER_START= ML_REFEREE_FINDER_RESULT= # Temp var for metering comms between nextjs api backend/nodes backend diff --git a/desci-server/kubernetes/deployment_dev.yaml b/desci-server/kubernetes/deployment_dev.yaml index f07689159..d63b26004 100644 --- a/desci-server/kubernetes/deployment_dev.yaml +++ b/desci-server/kubernetes/deployment_dev.yaml @@ -114,7 +114,7 @@ spec: export AWS_SQS_DATA_MIGRATION_QUEUE_URL="{{ .Data.AWS_SQS_DATA_MIGRATION_QUEUE_URL }}" export AWS_SQS_ML_TOOL_QUEUE_URL="{{ .Data.AWS_SQS_ML_TOOL_QUEUE_URL }}" export ML_REFEREE_GET_PRESIGNED_URL="{{ .Data.ML_REFEREE_GET_PRESIGNED_URL }}" - export ML_REFEREE_TRIGGER_CID="{{ .Data.ML_REFEREE_TRIGGER_CID }}" + export ML_REFEREE_TRIGGER_START="{{ .Data.ML_REFEREE_TRIGGER_START }}" export ML_REFEREE_FINDER_RESULT="{{ .Data.ML_REFEREE_FINDER_RESULT }}" export ENABLE_GUEST_MODE="{{ .Data.ENABLE_GUEST_MODE }}" export MIXPANEL_TOKEN="{{ .Data.MIXPANEL_TOKEN }}" diff --git a/desci-server/kubernetes/deployment_prod.yaml b/desci-server/kubernetes/deployment_prod.yaml index 832b28cfc..a608194ac 100755 --- a/desci-server/kubernetes/deployment_prod.yaml +++ b/desci-server/kubernetes/deployment_prod.yaml @@ -113,7 +113,7 @@ spec: export AWS_SQS_DATA_MIGRATION_QUEUE_URL="{{ .Data.AWS_SQS_DATA_MIGRATION_QUEUE_URL }}" export AWS_SQS_ML_TOOL_QUEUE_URL="{{ .Data.AWS_SQS_ML_TOOL_QUEUE_URL }}" export ML_REFEREE_GET_PRESIGNED_URL="{{ .Data.ML_REFEREE_GET_PRESIGNED_URL }}" - export ML_REFEREE_TRIGGER_CID="{{ .Data.ML_REFEREE_TRIGGER_CID }}" + export ML_REFEREE_TRIGGER_START="{{ .Data.ML_REFEREE_TRIGGER_START }}" export ML_REFEREE_FINDER_RESULT="{{ .Data.ML_REFEREE_FINDER_RESULT }}" export ENABLE_GUEST_MODE="{{ .Data.ENABLE_GUEST_MODE }}" export MIXPANEL_TOKEN="{{ .Data.MIXPANEL_TOKEN }}" diff --git a/desci-server/kubernetes/deployment_staging.yaml b/desci-server/kubernetes/deployment_staging.yaml index 8c5de8388..3a9c5d2e3 100644 --- a/desci-server/kubernetes/deployment_staging.yaml +++ b/desci-server/kubernetes/deployment_staging.yaml @@ -120,7 +120,7 @@ spec: export AWS_SQS_DATA_MIGRATION_QUEUE_URL="{{ .Data.AWS_SQS_DATA_MIGRATION_QUEUE_URL }}" export AWS_SQS_ML_TOOL_QUEUE_URL="{{ .Data.AWS_SQS_ML_TOOL_QUEUE_URL }}" export ML_REFEREE_GET_PRESIGNED_URL="{{ .Data.ML_REFEREE_GET_PRESIGNED_URL }}" - export ML_REFEREE_TRIGGER_CID="{{ .Data.ML_REFEREE_TRIGGER_CID }}" + export ML_REFEREE_TRIGGER_START="{{ .Data.ML_REFEREE_TRIGGER_START }}" export ML_REFEREE_FINDER_RESULT="{{ .Data.ML_REFEREE_FINDER_RESULT }}" export ENABLE_GUEST_MODE="{{ .Data.ENABLE_GUEST_MODE }}" export MIXPANEL_TOKEN="{{ .Data.MIXPANEL_TOKEN }}" From 061c51608bf6762d773e83734e8fbbd792c42f44 Mon Sep 17 00:00:00 2001 From: kadami <86646883+kadamidev@users.noreply.github.com> Date: Wed, 20 Aug 2025 06:41:38 +1000 Subject: [PATCH 04/10] add api file prefix var --- .env.example | 1 + desci-server/kubernetes/deployment_dev.yaml | 1 + desci-server/kubernetes/deployment_prod.yaml | 1 + desci-server/kubernetes/deployment_staging.yaml | 1 + 4 files changed, 4 insertions(+) diff --git a/.env.example b/.env.example index 4713efdb8..1ed78b3a4 100755 --- a/.env.example +++ b/.env.example @@ -205,6 +205,7 @@ MIXPANEL_TOKEN= ML_REFEREE_GET_PRESIGNED_URL= ML_REFEREE_TRIGGER_START= ML_REFEREE_FINDER_RESULT= +ML_REFEREE_FILE_PREFIX= # Temp var for metering comms between nextjs api backend/nodes backend INTERNAL_SERVICE_SECRET=psssssssssst \ No newline at end of file diff --git a/desci-server/kubernetes/deployment_dev.yaml b/desci-server/kubernetes/deployment_dev.yaml index d63b26004..e237c9520 100644 --- a/desci-server/kubernetes/deployment_dev.yaml +++ b/desci-server/kubernetes/deployment_dev.yaml @@ -116,6 +116,7 @@ spec: export ML_REFEREE_GET_PRESIGNED_URL="{{ .Data.ML_REFEREE_GET_PRESIGNED_URL }}" export ML_REFEREE_TRIGGER_START="{{ .Data.ML_REFEREE_TRIGGER_START }}" export ML_REFEREE_FINDER_RESULT="{{ .Data.ML_REFEREE_FINDER_RESULT }}" + export ML_REFEREE_FILE_PREFIX="{{ .Data.ML_REFEREE_FILE_PREFIX }}" export ENABLE_GUEST_MODE="{{ .Data.ENABLE_GUEST_MODE }}" export MIXPANEL_TOKEN="{{ .Data.MIXPANEL_TOKEN }}" export DEBUG_TEST=0; diff --git a/desci-server/kubernetes/deployment_prod.yaml b/desci-server/kubernetes/deployment_prod.yaml index a608194ac..e9ace35a7 100755 --- a/desci-server/kubernetes/deployment_prod.yaml +++ b/desci-server/kubernetes/deployment_prod.yaml @@ -115,6 +115,7 @@ spec: export ML_REFEREE_GET_PRESIGNED_URL="{{ .Data.ML_REFEREE_GET_PRESIGNED_URL }}" export ML_REFEREE_TRIGGER_START="{{ .Data.ML_REFEREE_TRIGGER_START }}" export ML_REFEREE_FINDER_RESULT="{{ .Data.ML_REFEREE_FINDER_RESULT }}" + export ML_REFEREE_FILE_PREFIX="{{ .Data.ML_REFEREE_FILE_PREFIX }}" export ENABLE_GUEST_MODE="{{ .Data.ENABLE_GUEST_MODE }}" export MIXPANEL_TOKEN="{{ .Data.MIXPANEL_TOKEN }}" export IGNORE_LINE=0; diff --git a/desci-server/kubernetes/deployment_staging.yaml b/desci-server/kubernetes/deployment_staging.yaml index 3a9c5d2e3..ab1245907 100644 --- a/desci-server/kubernetes/deployment_staging.yaml +++ b/desci-server/kubernetes/deployment_staging.yaml @@ -122,6 +122,7 @@ spec: export ML_REFEREE_GET_PRESIGNED_URL="{{ .Data.ML_REFEREE_GET_PRESIGNED_URL }}" export ML_REFEREE_TRIGGER_START="{{ .Data.ML_REFEREE_TRIGGER_START }}" export ML_REFEREE_FINDER_RESULT="{{ .Data.ML_REFEREE_FINDER_RESULT }}" + export ML_REFEREE_FILE_PREFIX="{{ .Data.ML_REFEREE_FILE_PREFIX }}" export ENABLE_GUEST_MODE="{{ .Data.ENABLE_GUEST_MODE }}" export MIXPANEL_TOKEN="{{ .Data.MIXPANEL_TOKEN }}" export DEBUG_TEST=0; From 7df6edc63b7f02ae35c629f0cf57497e992c980f Mon Sep 17 00:00:00 2001 From: kadami <86646883+kadamidev@users.noreply.github.com> Date: Wed, 20 Aug 2025 06:47:25 +1000 Subject: [PATCH 05/10] ref finder services updated for updated API, still needs more sqsHandler adjustments --- .../externalApi/RefereeRecommenderService.ts | 109 +++++++++++------- .../RefereeRecommenderSqsHandler.ts | 5 +- 2 files changed, 71 insertions(+), 43 deletions(-) diff --git a/desci-server/src/services/externalApi/RefereeRecommenderService.ts b/desci-server/src/services/externalApi/RefereeRecommenderService.ts index 59dc58c54..d1b9fe764 100644 --- a/desci-server/src/services/externalApi/RefereeRecommenderService.ts +++ b/desci-server/src/services/externalApi/RefereeRecommenderService.ts @@ -1,7 +1,6 @@ import { randomUUID } from 'crypto'; +import { createHash } from 'crypto'; -import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; -import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { ExternalApi, Feature } from '@prisma/client'; import axios from 'axios'; import { ok, err, Result } from 'neverthrow'; @@ -15,21 +14,12 @@ const logger = parentLogger.child({ module: 'RefereeRecommender::Service' }); export const SESSION_TTL_SECONDS = 86400; // 24 hours -// Initialize S3 client and config at module level -const s3Client = new S3Client({ - region: process.env.AWS_REGION || 'us-east-2', - credentials: { - accessKeyId: process.env.AWS_ACCESS_KEY_ID || '', - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || '', - }, -}); - -const BUCKET_NAME = process.env.REFEREE_RECOMMENDER_S3_BUCKET || ''; const API_VERSION = process.env.REFEREE_RECOMMENDER_VERSION || '0.1.3'; // ML Referee API endpoints -const ML_REFEREE_TRIGGER_URL = process.env.ML_REFEREE_TRIGGER_CID; +const ML_REFEREE_TRIGGER_URL = process.env.ML_REFEREE_TRIGGER_START; const ML_REFEREE_RESULTS_URL = process.env.ML_REFEREE_FINDER_RESULT; +const ML_REFEREE_GET_PRESIGNED_URL = process.env.ML_REFEREE_GET_PRESIGNED_URL; interface PresignedUrlRequest { userId: number; @@ -38,7 +28,7 @@ interface PresignedUrlRequest { interface RefereeRecommenderSession { userId: number; - originalFileName: string; + hashedFileUrl: string; createdAt: number; } @@ -48,8 +38,10 @@ interface PresignedUrlResponse { } interface TriggerRefereeRequest { - cid: string; - external?: boolean; + fileUrl: string; + originalFileName: string; + fileHash?: string; + hashIsVerified?: boolean; top_n_closely_matching?: number; number_referees?: number; force_run?: boolean; @@ -66,7 +58,6 @@ interface TriggerRefereeRequest { } interface TriggerRefereeResponse { - execution_arn?: string; uploaded_file_name: string; api_version: string; info: string; @@ -78,6 +69,17 @@ interface GetRefereeResultsResponse { result?: any; } +/** + * Hashes a file URL with SHA256, and encodes it in Base64 + * This is for a short cache key in redis for URLs, S3 file URLs can be around ~1500 chars otherwise. + * @param fileUrl - The file URL to hash + * @returns The Base64-encoded hash of the file URL + */ +function prepareCacheKey(fileUrl: string) { + // string -> SHA256 -> Base64 + return createHash('sha256').update(fileUrl).digest('base64'); // ~44 chars with '=' padding +} + async function generatePresignedUploadUrl(request: PresignedUrlRequest): Promise> { try { // Check feature limits first @@ -120,25 +122,32 @@ async function generatePresignedUploadUrl(request: PresignedUrlRequest): Promise 'Feature limit check passed for presigned URL generation', ); - // Generate filename with convention: referee_rec_v0.1.3_{original_file.pdf} - // original file name is salted with a UUID atm, will change to a hash in the future. + // Call external endpoint to get presigned URL const fileName = generateFileName(request.originalFileName); - // Create presigned URL for upload - const command = new PutObjectCommand({ - Bucket: BUCKET_NAME, - Key: fileName, - ContentType: 'application/pdf', - }); + if (!ML_REFEREE_GET_PRESIGNED_URL) { + return err(new Error('ML_REFEREE_GET_PRESIGNED_URL environment variable not configured')); + } - const presignedUrl = await getSignedUrl(s3Client, command, { - expiresIn: 3600, // 1 hour + const presignedResponse = await axios.get(ML_REFEREE_GET_PRESIGNED_URL, { + params: { + file_name: fileName, + }, }); - // Store session in Redis - const storeResult = await storeSession(fileName, { + if (presignedResponse.status !== 200) { + return err(new Error(`Presigned URL service returned status ${presignedResponse.status}`)); + } + + const { upload_url, download_url, s3_file_name } = presignedResponse.data; + + // Use download_url for cache key and store hashedFileUrl in session + const hashedFileUrl = prepareCacheKey(download_url); + + // Store session in Redis using the hashed download URL + const storeResult = await storeSession(hashedFileUrl, { userId: request.userId, - originalFileName: request.originalFileName, + hashedFileUrl, createdAt: Date.now(), }); @@ -146,17 +155,19 @@ async function generatePresignedUploadUrl(request: PresignedUrlRequest): Promise return err(storeResult.error); } - await markSessionAsActive(request.userId, fileName); + await markSessionAsActive(request.userId, hashedFileUrl); logger.info( { userId: request.userId, - fileName, + originalFileName: request.originalFileName, + s3FileName: s3_file_name, + hashedFileUrl, }, - 'Generated presigned URL for referee recommender', + 'Generated presigned URL for referee recommender via external endpoint', ); - return ok({ presignedUrl, fileName }); + return ok({ presignedUrl: upload_url, downloadUrl: download_url, fileName: s3_file_name }); } catch (error) { logger.error({ error, request }, 'Failed to generate presigned URL'); return err(error instanceof Error ? error : new Error('Failed to generate presigned URL')); @@ -320,7 +331,14 @@ async function triggerRefereeRecommendation( 'Feature limit check passed for referee recommendation trigger', ); - logger.info({ cid: request.cid, external: request.external }, 'Triggering referee recommendation'); + logger.info( + { + fileUrl: request.fileUrl, + fileHash: request.fileHash, + hashIsVerified: request.hashIsVerified, + }, + 'Triggering referee recommendation', + ); const response = await axios.post(ML_REFEREE_TRIGGER_URL, request, { headers: { @@ -333,11 +351,14 @@ async function triggerRefereeRecommendation( return err(new Error(`ML Referee API returned status ${response.status}`)); } - // Store session for CID-based requests + // Store session for URL-based requests if (response.data.uploaded_file_name) { + // Use prepareCacheKey on the fileUrl for consistent hashing + const hashedFileUrl = prepareCacheKey(request.fileUrl); + const session: RefereeRecommenderSession = { userId, - originalFileName: `${request.cid}`, + hashedFileUrl, createdAt: Date.now(), }; @@ -345,14 +366,22 @@ async function triggerRefereeRecommendation( await markSessionAsActive(userId, response.data.uploaded_file_name); logger.debug( - { userId, cid: request.cid, uploadedFileName: response.data.uploaded_file_name }, - 'Stored CID-based session and marked as active', + { + userId, + fileUrl: request.fileUrl, + fileHash: request.fileHash, + hashedFileUrl, + uploadedFileName: response.data.uploaded_file_name, + }, + 'Stored URL-based session and marked as active', ); } logger.info( { - cid: request.cid, + fileUrl: request.fileUrl, + fileHash: request.fileHash, + hashIsVerified: request.hashIsVerified, uploaded_file_name: response.data.uploaded_file_name, info: response.data.info, }, diff --git a/desci-server/src/services/externalApi/RefereeRecommenderSqsHandler.ts b/desci-server/src/services/externalApi/RefereeRecommenderSqsHandler.ts index 4bf9f0604..a7fafcb57 100644 --- a/desci-server/src/services/externalApi/RefereeRecommenderSqsHandler.ts +++ b/desci-server/src/services/externalApi/RefereeRecommenderSqsHandler.ts @@ -1,4 +1,4 @@ -import { ExternalApi } from '@prisma/client'; +import { ExternalApi, Prisma } from '@prisma/client'; import { prisma } from '../../client.js'; import { logger as parentLogger } from '../../logger.js'; @@ -7,7 +7,6 @@ import { sqsService } from '../sqs/SqsService.js'; import { emitWebsocketEvent, WebSocketEventType } from '../websocketService.js'; import { RefereeRecommenderService, SESSION_TTL_SECONDS } from './RefereeRecommenderService.js'; -import { JsonValue } from 'aws-sdk/clients/appmesh.js'; const logger = parentLogger.child({ module: 'RefereeRecommender::SqsHandler' }); @@ -265,7 +264,7 @@ export class RefereeRecommenderSqsHandler { data: { userId: session.userId, apiType: ExternalApi.REFEREE_FINDER, - queryingData: queryingData as unknown as JsonValue, // Fast queries on this field + queryingData: queryingData as unknown as Prisma.JsonValue, // Fast queries on this field data: metadataData, // Full metadata storage }, }); From ff39b6372b51ab5adc67c547bc686504acc3ba79 Mon Sep 17 00:00:00 2001 From: kadami <86646883+kadamidev@users.noreply.github.com> Date: Wed, 20 Aug 2025 07:00:13 +1000 Subject: [PATCH 06/10] controller/schema updates for changes --- .../RefereeRecommender/issuePresignedUrl.ts | 4 +- .../triggerRecommendation.ts | 67 +++++++++++++++++-- .../src/schemas/externalApi.schema.ts | 4 +- .../externalApi/RefereeRecommenderService.ts | 25 ++++--- .../src/services/sqs/SqsMessageTypes.ts | 1 + 5 files changed, 78 insertions(+), 23 deletions(-) diff --git a/desci-server/src/controllers/externalApi/RefereeRecommender/issuePresignedUrl.ts b/desci-server/src/controllers/externalApi/RefereeRecommender/issuePresignedUrl.ts index 06eb395fb..803afecb9 100644 --- a/desci-server/src/controllers/externalApi/RefereeRecommender/issuePresignedUrl.ts +++ b/desci-server/src/controllers/externalApi/RefereeRecommender/issuePresignedUrl.ts @@ -1,12 +1,12 @@ +import { ExternalApi } from '@prisma/client'; import { Response } from 'express'; import { z } from 'zod'; +import { prisma } from '../../../client.js'; import { sendSuccess, sendError } from '../../../core/api.js'; import { AuthenticatedRequest } from '../../../core/types.js'; import { logger as parentLogger } from '../../../logger.js'; import { RefereeRecommenderService } from '../../../services/externalApi/RefereeRecommenderService.js'; -import { prisma } from '../../../client.js'; -import { ExternalApi } from '@prisma/client'; const logger = parentLogger.child({ module: 'RefereeRecommender::PresignedUrlController' }); diff --git a/desci-server/src/controllers/externalApi/RefereeRecommender/triggerRecommendation.ts b/desci-server/src/controllers/externalApi/RefereeRecommender/triggerRecommendation.ts index c69a901a2..f51dac9c7 100644 --- a/desci-server/src/controllers/externalApi/RefereeRecommender/triggerRecommendation.ts +++ b/desci-server/src/controllers/externalApi/RefereeRecommender/triggerRecommendation.ts @@ -1,6 +1,8 @@ +import { ExternalApi } from '@prisma/client'; import { Response } from 'express'; import { z } from 'zod'; +import { prisma } from '../../../client.js'; import { sendSuccess, sendError } from '../../../core/api.js'; import { AuthenticatedRequest } from '../../../core/types.js'; import { logger as parentLogger } from '../../../logger.js'; @@ -19,18 +21,69 @@ export const triggerRecommendation = async (req: AuthenticatedRequest, res: Resp } const { body } = triggerRefereeRecommendationSchema.parse(req); + const { fileUrl, fileHash } = body; logger.info( { userId: user.id, - cid: body.cid, - external: body.external, + fileUrl, + fileHash, }, - 'Triggering referee recommendation', + 'Checking cache and potentially triggering referee recommendation', ); + // Check if we have cached results for this fileHash (if provided) + let cachedResult = null; + + if (fileHash) { + cachedResult = await prisma.externalApiUsage.findFirst({ + where: { + userId: user.id, + apiType: ExternalApi.REFEREE_FINDER, + queryingData: { + path: ['fileHash'], + equals: fileHash, + }, + }, + orderBy: { + createdAt: 'desc', // Get the most recent result + }, + }); + + if (cachedResult) { + logger.info( + { + userId: user.id, + fileHash, + cachedResultId: cachedResult.id, + }, + 'Found cached results for fileHash, returning cached data', + ); + + return sendSuccess(res, { + cached: true, + message: 'Results for this file are already available', + createdAt: cachedResult.createdAt, + fileHash: fileHash, // TODO: Add formatted cache key + }); + } + } + const result = await RefereeRecommenderService.triggerRefereeRecommendation( - body as typeof body & { cid: string }, + { + // hash_value, + // hash_verified, + file_url: body.fileUrl, + top_n_closely_matching: body.top_n_closely_matching, + number_referees: body.number_referees, + force_run: body.force_run, + classify: body.classify, + coi_filter: body.coi_filter, + meta_data_only: body.meta_data_only, + exclude_fields: body.exclude_fields, + exclude_works: body.exclude_works, + exclude_authors: body.exclude_authors, + }, user.id, ); @@ -39,7 +92,6 @@ export const triggerRecommendation = async (req: AuthenticatedRequest, res: Resp { error: result.error, userId: user.id, - cid: body.cid, }, 'Failed to trigger referee recommendation', ); @@ -60,7 +112,10 @@ export const triggerRecommendation = async (req: AuthenticatedRequest, res: Resp 'Successfully triggered referee recommendation', ); - return sendSuccess(res, responseData); + return sendSuccess(res, { + ...responseData, + cached: false, + }); } catch (error) { if (error instanceof z.ZodError) { logger.warn({ error: error.errors }, 'Invalid request parameters'); diff --git a/desci-server/src/schemas/externalApi.schema.ts b/desci-server/src/schemas/externalApi.schema.ts index d2ea6282e..7126986c4 100644 --- a/desci-server/src/schemas/externalApi.schema.ts +++ b/desci-server/src/schemas/externalApi.schema.ts @@ -14,8 +14,8 @@ export const generatePresignedUrlSchema = z.object({ // Trigger Referee Recommendation Request export const triggerRefereeRecommendationSchema = z.object({ body: z.object({ - cid: z.string().min(1, 'CID is required'), - external: z.boolean().optional().default(false), + fileUrl: z.string().min(1, 'fileUrl is required'), + fileHash: z.string().min(1, 'fileHash is required').optional(), top_n_closely_matching: z.number().int().min(1).max(50).optional().default(5), number_referees: z.number().int().min(1).max(50).optional().default(10), force_run: z.boolean().optional().default(false), diff --git a/desci-server/src/services/externalApi/RefereeRecommenderService.ts b/desci-server/src/services/externalApi/RefereeRecommenderService.ts index d1b9fe764..3b207ae74 100644 --- a/desci-server/src/services/externalApi/RefereeRecommenderService.ts +++ b/desci-server/src/services/externalApi/RefereeRecommenderService.ts @@ -38,10 +38,9 @@ interface PresignedUrlResponse { } interface TriggerRefereeRequest { - fileUrl: string; - originalFileName: string; - fileHash?: string; - hashIsVerified?: boolean; + file_url: string; + hash_value?: string; + hash_verified?: boolean; top_n_closely_matching?: number; number_referees?: number; force_run?: boolean; @@ -333,9 +332,9 @@ async function triggerRefereeRecommendation( logger.info( { - fileUrl: request.fileUrl, - fileHash: request.fileHash, - hashIsVerified: request.hashIsVerified, + file_url: request.file_url, + hash_value: request.hash_value, + hash_verified: request.hash_verified, }, 'Triggering referee recommendation', ); @@ -354,7 +353,7 @@ async function triggerRefereeRecommendation( // Store session for URL-based requests if (response.data.uploaded_file_name) { // Use prepareCacheKey on the fileUrl for consistent hashing - const hashedFileUrl = prepareCacheKey(request.fileUrl); + const hashedFileUrl = prepareCacheKey(request.file_url); const session: RefereeRecommenderSession = { userId, @@ -368,8 +367,8 @@ async function triggerRefereeRecommendation( logger.debug( { userId, - fileUrl: request.fileUrl, - fileHash: request.fileHash, + file_url: request.file_url, + hash_value: request.hash_value, hashedFileUrl, uploadedFileName: response.data.uploaded_file_name, }, @@ -379,9 +378,9 @@ async function triggerRefereeRecommendation( logger.info( { - fileUrl: request.fileUrl, - fileHash: request.fileHash, - hashIsVerified: request.hashIsVerified, + file_url: request.file_url, + hash_value: request.hash_value, + hash_verified: request.hash_verified, uploaded_file_name: response.data.uploaded_file_name, info: response.data.info, }, diff --git a/desci-server/src/services/sqs/SqsMessageTypes.ts b/desci-server/src/services/sqs/SqsMessageTypes.ts index f4dc0f825..4c8b53835 100644 --- a/desci-server/src/services/sqs/SqsMessageTypes.ts +++ b/desci-server/src/services/sqs/SqsMessageTypes.ts @@ -29,5 +29,6 @@ export interface ExternalApiSqsMessage extends BaseSqsMessage { eventType: 'PROCESSING_STARTED' | 'PROCESSING_COMPLETED' | 'PROCESSING_FAILED' | 'PROCESSING_PROGRESS'; file_name: string; // e.g., "referee_rec_v0.1.3_{original_file.pdf}" // Move to hashes later for security. file_hash?: string; // SHA-256 hash of the file content + file_url: string; // Source URL data?: any; } From 2cd5e348c0a6185d1af336e5c02acb9579003fcb Mon Sep 17 00:00:00 2001 From: kadami <86646883+kadamidev@users.noreply.github.com> Date: Wed, 20 Aug 2025 20:56:10 +1000 Subject: [PATCH 07/10] fix up bugs, more changes for API update --- .../RefereeRecommender/issuePresignedUrl.ts | 7 +- .../triggerRecommendation.ts | 9 ++- .../externalApi/RefereeRecommenderService.ts | 80 ++++++++++++------- .../RefereeRecommenderSqsHandler.ts | 5 +- .../src/services/sqs/SqsMessageTypes.ts | 9 +++ 5 files changed, 74 insertions(+), 36 deletions(-) diff --git a/desci-server/src/controllers/externalApi/RefereeRecommender/issuePresignedUrl.ts b/desci-server/src/controllers/externalApi/RefereeRecommender/issuePresignedUrl.ts index 803afecb9..3e7ae8949 100644 --- a/desci-server/src/controllers/externalApi/RefereeRecommender/issuePresignedUrl.ts +++ b/desci-server/src/controllers/externalApi/RefereeRecommender/issuePresignedUrl.ts @@ -66,8 +66,7 @@ export const generatePresignedUrl = async (req: AuthenticatedRequest, res: Respo return sendSuccess(res, { cached: true, message: 'Results for this file are already available', - resultId: cachedResult.id, - createdAt: cachedResult.createdAt, + resultKey: RefereeRecommenderService.prepareFormattedFileName(fileHash), }); } @@ -87,7 +86,7 @@ export const generatePresignedUrl = async (req: AuthenticatedRequest, res: Respo return sendError(res, 'Failed to generate presigned URL', 500); } - const { presignedUrl, fileName: generatedFileName } = result.value; + const { presignedUrl, downloadUrl, fileName: generatedFileName } = result.value; logger.info( { @@ -100,9 +99,9 @@ export const generatePresignedUrl = async (req: AuthenticatedRequest, res: Respo return sendSuccess(res, { cached: false, presignedUrl, + downloadUrl, fileName: generatedFileName, fileHash, - expiresIn: 3600, // 1 hour }); } catch (error) { if (error instanceof z.ZodError) { diff --git a/desci-server/src/controllers/externalApi/RefereeRecommender/triggerRecommendation.ts b/desci-server/src/controllers/externalApi/RefereeRecommender/triggerRecommendation.ts index f51dac9c7..a08c14647 100644 --- a/desci-server/src/controllers/externalApi/RefereeRecommender/triggerRecommendation.ts +++ b/desci-server/src/controllers/externalApi/RefereeRecommender/triggerRecommendation.ts @@ -1,5 +1,6 @@ import { ExternalApi } from '@prisma/client'; import { Response } from 'express'; +import _ from 'lodash'; import { z } from 'zod'; import { prisma } from '../../../client.js'; @@ -64,7 +65,7 @@ export const triggerRecommendation = async (req: AuthenticatedRequest, res: Resp cached: true, message: 'Results for this file are already available', createdAt: cachedResult.createdAt, - fileHash: fileHash, // TODO: Add formatted cache key + resultKey: RefereeRecommenderService.prepareFormattedFileName(fileHash), }); } } @@ -106,14 +107,16 @@ export const triggerRecommendation = async (req: AuthenticatedRequest, res: Resp logger.info( { userId: user.id, - uploadedFileName: responseData.uploaded_file_name, + uploadedFileName: responseData.UploadedFileName, info: responseData.info, }, 'Successfully triggered referee recommendation', ); + const formattedResponse = _.omit(responseData, 'execution_arn'); + return sendSuccess(res, { - ...responseData, + ...formattedResponse, cached: false, }); } catch (error) { diff --git a/desci-server/src/services/externalApi/RefereeRecommenderService.ts b/desci-server/src/services/externalApi/RefereeRecommenderService.ts index 3b207ae74..222b67450 100644 --- a/desci-server/src/services/externalApi/RefereeRecommenderService.ts +++ b/desci-server/src/services/externalApi/RefereeRecommenderService.ts @@ -1,5 +1,4 @@ -import { randomUUID } from 'crypto'; -import { createHash } from 'crypto'; +import { randomUUID, createHash } from 'crypto'; import { ExternalApi, Feature } from '@prisma/client'; import axios from 'axios'; @@ -21,6 +20,8 @@ const ML_REFEREE_TRIGGER_URL = process.env.ML_REFEREE_TRIGGER_START; const ML_REFEREE_RESULTS_URL = process.env.ML_REFEREE_FINDER_RESULT; const ML_REFEREE_GET_PRESIGNED_URL = process.env.ML_REFEREE_GET_PRESIGNED_URL; +const ML_REFEREE_FILE_PREFIX = process.env.ML_REFEREE_FILE_PREFIX; + interface PresignedUrlRequest { userId: number; originalFileName: string; @@ -34,6 +35,7 @@ interface RefereeRecommenderSession { interface PresignedUrlResponse { presignedUrl: string; + downloadUrl: string; fileName: string; } @@ -57,9 +59,10 @@ interface TriggerRefereeRequest { } interface TriggerRefereeResponse { - uploaded_file_name: string; + UploadedFileName: string; api_version: string; info: string; + cached: boolean; } interface GetRefereeResultsResponse { @@ -74,11 +77,26 @@ interface GetRefereeResultsResponse { * @param fileUrl - The file URL to hash * @returns The Base64-encoded hash of the file URL */ -function prepareCacheKey(fileUrl: string) { +function prepareSessionCacheKey(fileUrl: string) { + // We use the file URLs here instead of the file hash, because it's the only + // deterministic link that we have available early on for every upload method (S3, CID, External pdf url) + // for us to be able to retrieve a result back. // string -> SHA256 -> Base64 return createHash('sha256').update(fileUrl).digest('base64'); // ~44 chars with '=' padding } +/** + * Results are stored in this format: {API}_{API_VERSION}_{SHA256FILE_HASH} + * @param fileHash - The SHA256 hash of the file URL + */ +function prepareFormattedFileName(fileHash: string) { + if (!ML_REFEREE_FILE_PREFIX) { + console.error('ML_REFEREE_FILE_PREFIX is not set'); + throw new Error('ML_REFEREE_FILE_PREFIX env is not set'); + } + return ML_REFEREE_FILE_PREFIX + fileHash; +} + async function generatePresignedUploadUrl(request: PresignedUrlRequest): Promise> { try { // Check feature limits first @@ -141,7 +159,7 @@ async function generatePresignedUploadUrl(request: PresignedUrlRequest): Promise const { upload_url, download_url, s3_file_name } = presignedResponse.data; // Use download_url for cache key and store hashedFileUrl in session - const hashedFileUrl = prepareCacheKey(download_url); + const hashedFileUrl = prepareSessionCacheKey(download_url); // Store session in Redis using the hashed download URL const storeResult = await storeSession(hashedFileUrl, { @@ -186,13 +204,14 @@ function generateFileName(originalFileName: string): string { return `referee_rec_v${API_VERSION}_${uuid}_${sanitizedBaseName}.${fileExtension}`; } -async function storeSession(fileName: string, session: RefereeRecommenderSession): Promise> { +async function storeSession(fileUrl: string, session: RefereeRecommenderSession): Promise> { try { - const userCacheKey = `referee-recommender-session:${session.userId}:${fileName}`; + const hashedFileUrl = prepareSessionCacheKey(fileUrl); + const userCacheKey = `referee-recommender-session:${session.userId}:${hashedFileUrl}`; await setToCache(userCacheKey, session, SESSION_TTL_SECONDS); // Store/append to filename-based lookup for SQS handler - const filenameCacheKey = `referee-recommender-filename:${fileName}`; + const filenameCacheKey = `referee-recommender-file-url:${hashedFileUrl}`; const existingSessions = (await getFromCache(filenameCacheKey)) || []; const now = Date.now(); @@ -210,7 +229,7 @@ async function storeSession(fileName: string, session: RefereeRecommenderSession logger.debug( { - fileName, + fileUrl, userId: session.userId, sessionCount: validSessions.length, }, @@ -218,7 +237,7 @@ async function storeSession(fileName: string, session: RefereeRecommenderSession ); return ok(undefined); } catch (error) { - logger.error({ error, fileName }, 'Failed to store session in Redis'); + logger.error({ error, fileUrl }, 'Failed to store session in Redis'); return err(error instanceof Error ? error : new Error('Failed to store session in Redis')); } } @@ -241,9 +260,10 @@ async function getSession(fileName: string, userId: number): Promise> { +async function getSessionsByFileUrl(fileUrl: string): Promise> { try { - const filenameCacheKey = `referee-recommender-filename:${fileName}`; + const hashedFileUrl = prepareSessionCacheKey(fileUrl); + const filenameCacheKey = `referee-recommender-file-url:${fileUrl}`; const sessions = await getFromCache(filenameCacheKey); if (sessions && sessions.length > 0) { @@ -264,7 +284,7 @@ async function getSessionsByFileName(fileName: string): Promise { +async function markSessionAsActive(userId: number, fileUrl: string): Promise { try { + const hashedFileUrl = prepareSessionCacheKey(fileUrl); const activeSessionsKey = `referee-recommender-active:${userId}`; const activeSessions = (await getFromCache(activeSessionsKey)) || []; // Add this filename to active sessions if not already present - if (!activeSessions.includes(fileName)) { - activeSessions.push(fileName); + if (!activeSessions.includes(hashedFileUrl)) { + activeSessions.push(hashedFileUrl); await setToCache(activeSessionsKey, activeSessions, SESSION_TTL_SECONDS); } } catch (error) { - logger.error({ error, userId, fileName }, 'Failed to mark session as active'); + logger.error({ error, userId, fileUrl }, 'Failed to mark session as active'); } } -async function markSessionAsCompleted(userId: number, fileName: string): Promise { +async function markSessionAsCompleted(userId: number, fileUrl: string): Promise { try { + const hashedFileUrl = prepareSessionCacheKey(fileUrl); const activeSessionsKey = `referee-recommender-active:${userId}`; const activeSessions = (await getFromCache(activeSessionsKey)) || []; // Remove this filename from active sessions - const updatedSessions = activeSessions.filter((session) => session !== fileName); + const updatedSessions = activeSessions.filter((session) => session !== hashedFileUrl); if (updatedSessions.length > 0) { await setToCache(activeSessionsKey, updatedSessions, SESSION_TTL_SECONDS); @@ -578,7 +600,7 @@ async function markSessionAsCompleted(userId: number, fileName: string): Promise await setToCache(activeSessionsKey, [], 1); // Short TTL to effectively delete } } catch (error) { - logger.error({ error, userId, fileName }, 'Failed to mark session as completed'); + logger.error({ error, userId, fileUrl }, 'Failed to mark session as completed'); } } @@ -679,7 +701,7 @@ async function cleanupExpiredSessions(userId: number): Promise { export const RefereeRecommenderService = { generatePresignedUploadUrl, getSession, - getSessionsByFileName, + getSessionsByFileUrl, triggerRefereeRecommendation, getRefereeResults, getUserUsageCount, @@ -689,4 +711,6 @@ export const RefereeRecommenderService = { markSessionAsCompleted, handleProcessingFailure, cleanupExpiredSessions, + prepareFormattedFileName, + prepareCacheKey: prepareSessionCacheKey, }; diff --git a/desci-server/src/services/externalApi/RefereeRecommenderSqsHandler.ts b/desci-server/src/services/externalApi/RefereeRecommenderSqsHandler.ts index a7fafcb57..33b949a74 100644 --- a/desci-server/src/services/externalApi/RefereeRecommenderSqsHandler.ts +++ b/desci-server/src/services/externalApi/RefereeRecommenderSqsHandler.ts @@ -80,8 +80,11 @@ export class RefereeRecommenderSqsHandler { '[SQS] Processing referee recommender event', ); + debugger; + const fileUrl = eventData.data.file_url; + // Look up user sessions from Redis using filename - const sessionsResult = await RefereeRecommenderService.getSessionsByFileName(eventData.file_name); + const sessionsResult = await RefereeRecommenderService.getSessionsByFileUrl(fileUrl); if (sessionsResult.isErr()) { logger.warn( { fileName: eventData.file_name, error: sessionsResult.error.message }, diff --git a/desci-server/src/services/sqs/SqsMessageTypes.ts b/desci-server/src/services/sqs/SqsMessageTypes.ts index 4c8b53835..015d51c41 100644 --- a/desci-server/src/services/sqs/SqsMessageTypes.ts +++ b/desci-server/src/services/sqs/SqsMessageTypes.ts @@ -32,3 +32,12 @@ export interface ExternalApiSqsMessage extends BaseSqsMessage { file_url: string; // Source URL data?: any; } + +export type ExternalApiSqsMessageData = { + file_url: string; + hash_value?: string; + hash_verified?: boolean; + paper_title?: string; + api_version?: string; + completed_at?: number; // Unix TS +}; From 481efe329bb36740ad4b172ac2bf716d6c29a797 Mon Sep 17 00:00:00 2001 From: kadami <86646883+kadamidev@users.noreply.github.com> Date: Thu, 21 Aug 2025 08:44:52 +1000 Subject: [PATCH 08/10] latest fixes --- desci-server/src/docs/aiServices.ts | 339 ++++++++++++++++++ desci-server/src/docs/openai.ts | 3 + .../externalApi/RefereeRecommenderService.ts | 2 +- .../RefereeRecommenderSqsHandler.ts | 10 + .../src/services/sqs/SqsMessageTypes.ts | 3 +- 5 files changed, 355 insertions(+), 2 deletions(-) create mode 100644 desci-server/src/docs/aiServices.ts diff --git a/desci-server/src/docs/aiServices.ts b/desci-server/src/docs/aiServices.ts new file mode 100644 index 000000000..95e7e0d15 --- /dev/null +++ b/desci-server/src/docs/aiServices.ts @@ -0,0 +1,339 @@ +import { z } from 'zod'; +import { ZodOpenApiOperationObject, ZodOpenApiPathsObject } from 'zod-openapi'; + +// Schemas +const FileHashSchema = z.object({ + fileHash: z.string().min(1).describe('SHA-256 hash of the file content'), + fileName: z.string().min(1).describe('Original filename'), +}); + +const PresignedUrlResponseSchema = z.object({ + cached: z.boolean().describe('Whether results are cached'), + presignedUrl: z.string().optional().describe('S3 presigned upload URL'), + downloadUrl: z.string().optional().describe('S3 download URL'), + fileName: z.string().optional().describe('Generated S3 filename'), + fileHash: z.string().optional().describe('File hash'), + resultKey: z.string().optional().describe('Formatted cache key for results'), + message: z.string().optional().describe('Cache message'), +}); + +const TriggerRefereeRequestSchema = z.object({ + fileUrl: z.string().url().describe('URL to the PDF file'), + fileHash: z.string().optional().describe('SHA-256 hash of file (for caching)'), + top_n_closely_matching: z + .number() + .int() + .min(1) + .max(50) + .optional() + .default(5) + .describe('Number of closely matching papers to analyze'), + number_referees: z + .number() + .int() + .min(1) + .max(50) + .optional() + .default(10) + .describe('Number of referee recommendations to return'), + force_run: z.boolean().optional().default(false).describe('Force re-processing even if cached results exist'), + classify: z.boolean().optional().describe('Enable classification features'), + coi_filter: z + .object({ + co_author_overlap: z.boolean().optional(), + institution_overlap: z.boolean().optional(), + supervisor_supervisee_check: z.boolean().optional(), + }) + .optional() + .describe('Conflict of interest filtering options'), + meta_data_only: z.boolean().optional().describe('Return only metadata'), + exclude_fields: z.array(z.string()).optional().describe('Research fields to exclude'), + exclude_works: z.array(z.string()).optional().describe('Specific works to exclude'), + exclude_authors: z.array(z.string()).optional().describe('Specific authors to exclude'), +}); + +const TriggerRefereeResponseSchema = z.object({ + cached: z.boolean().describe('Whether results were cached'), + UploadedFileName: z.string().optional().describe('Backend filename for processing'), + api_version: z.string().optional().describe('API version used'), + info: z.string().optional().describe('Processing information'), + resultKey: z.string().optional().describe('Formatted cache key'), + fileHash: z.string().optional().describe('File hash'), + createdAt: z.string().optional().describe('Creation timestamp'), + data: z.any().optional().describe('Cached result data'), +}); + +const UsageStatusSchema = z.object({ + totalLimit: z.number().nullable().describe('Total monthly limit (null for unlimited)'), + totalUsed: z.number().describe('Total requests used this month'), + totalRemaining: z.number().nullable().describe('Remaining requests (null for unlimited)'), + planCodename: z.string().describe('User plan identifier'), + isWithinLimit: z.boolean().describe('Whether user is within usage limits'), +}); + +const RefereeResultsSchema = z.object({ + ok: z.boolean().describe('Request success status'), + data: z + .object({ + paper_data: z.object({ + abstract: z.string(), + title: z.string(), + author_info: z.array( + z.object({ + display_name: z.string(), + }), + ), + context_novelty: z.number(), + content_novelty: z.number(), + }), + focal_authors_data: z.array( + z.object({ + display_name: z.string(), + id: z.string(), + orcid: z.string(), + }), + ), + referees: z.object({ + recommended: z.array( + z.object({ + display_name: z.string(), + id: z.string(), + orcid: z.string(), + cited_by_count: z.number(), + affiliations: z.string(), + h_index: z.number(), + tags: z.array(z.string()), + works: z.array( + z.object({ + work_id: z.string(), + title: z.string(), + doi: z.string(), + similarity_score: z.number(), + }), + ), + work_similarity_score: z.number(), + topic_similarity_score: z.number(), + total_direct_citations: z.number().nullable(), + earliest_work: z.number(), + }), + ), + }), + }) + .describe('Referee recommendation results'), +}); + +const ErrorResponseSchema = z.object({ + ok: z.boolean().default(false), + message: z.string().describe('Error message'), +}); + +// Operations +export const generatePresignedUrlOperation: ZodOpenApiOperationObject = { + operationId: 'generatePresignedUrl', + tags: ['AI Services'], + summary: 'Generate presigned URL for referee recommender file upload', + description: + 'Get a presigned S3 URL for uploading a PDF file for referee recommendation analysis. Checks for cached results first.', + security: [{ bearerAuth: [] }], + requestBody: { + content: { + 'application/json': { + schema: FileHashSchema, + }, + }, + }, + responses: { + '200': { + description: 'Presigned URL generated successfully or cached results found', + content: { + 'application/json': { + schema: PresignedUrlResponseSchema, + }, + }, + }, + '400': { + description: 'Invalid request parameters', + content: { + 'application/json': { + schema: ErrorResponseSchema, + }, + }, + }, + '401': { + description: 'Authentication required', + content: { + 'application/json': { + schema: ErrorResponseSchema, + }, + }, + }, + '500': { + description: 'Internal server error', + content: { + 'application/json': { + schema: ErrorResponseSchema, + }, + }, + }, + }, +}; + +export const triggerRefereeRecommendationOperation: ZodOpenApiOperationObject = { + operationId: 'triggerRefereeRecommendation', + tags: ['AI Services'], + summary: 'Trigger referee recommendation analysis', + description: + 'Start processing a PDF file for referee recommendations. Supports caching and various filtering options.', + security: [{ bearerAuth: [] }], + requestBody: { + content: { + 'application/json': { + schema: TriggerRefereeRequestSchema, + }, + }, + }, + responses: { + '200': { + description: 'Processing triggered successfully or cached results returned', + content: { + 'application/json': { + schema: TriggerRefereeResponseSchema, + }, + }, + }, + '400': { + description: 'Invalid request parameters', + content: { + 'application/json': { + schema: ErrorResponseSchema, + }, + }, + }, + '401': { + description: 'Authentication required', + content: { + 'application/json': { + schema: ErrorResponseSchema, + }, + }, + }, + '429': { + description: 'Rate limit exceeded', + content: { + 'application/json': { + schema: ErrorResponseSchema, + }, + }, + }, + '500': { + description: 'Internal server error', + content: { + 'application/json': { + schema: ErrorResponseSchema, + }, + }, + }, + }, +}; + +export const getRefereeResultsOperation: ZodOpenApiOperationObject = { + operationId: 'getRefereeResults', + tags: ['AI Services'], + summary: 'Get referee recommendation results', + description: 'Retrieve the results of referee recommendation analysis using the uploaded filename.', + security: [{ bearerAuth: [] }], + parameters: [ + { + name: 'UploadedFileName', + in: 'query', + required: true, + description: 'Backend filename from processing', + schema: { + type: 'string', + }, + }, + ], + responses: { + '200': { + description: 'Results retrieved successfully', + content: { + 'application/json': { + schema: RefereeResultsSchema, + }, + }, + }, + '404': { + description: 'Results not found', + content: { + 'application/json': { + schema: ErrorResponseSchema, + }, + }, + }, + '401': { + description: 'Authentication required', + content: { + 'application/json': { + schema: ErrorResponseSchema, + }, + }, + }, + '500': { + description: 'Internal server error', + content: { + 'application/json': { + schema: ErrorResponseSchema, + }, + }, + }, + }, +}; + +export const getRefereeUsageOperation: ZodOpenApiOperationObject = { + operationId: 'getRefereeUsage', + tags: ['AI Services'], + summary: 'Get referee recommender usage status', + description: 'Check current usage limits and remaining quota for referee recommender service.', + security: [{ bearerAuth: [] }], + responses: { + '200': { + description: 'Usage status retrieved successfully', + content: { + 'application/json': { + schema: UsageStatusSchema, + }, + }, + }, + '401': { + description: 'Authentication required', + content: { + 'application/json': { + schema: ErrorResponseSchema, + }, + }, + }, + '500': { + description: 'Internal server error', + content: { + 'application/json': { + schema: ErrorResponseSchema, + }, + }, + }, + }, +}; + +export const aiServicesPaths: ZodOpenApiPathsObject = { + '/services/ai/referee-recommender/presigned-url': { + post: generatePresignedUrlOperation, + }, + '/services/ai/referee-recommender/trigger': { + post: triggerRefereeRecommendationOperation, + }, + '/services/ai/referee-recommender/results': { + get: getRefereeResultsOperation, + }, + '/services/ai/referee-recommender/usage': { + get: getRefereeUsageOperation, + }, +}; diff --git a/desci-server/src/docs/openai.ts b/desci-server/src/docs/openai.ts index a32acf588..e7ef203bf 100644 --- a/desci-server/src/docs/openai.ts +++ b/desci-server/src/docs/openai.ts @@ -30,6 +30,7 @@ import { updateSubmissionStatusOperation, } from './submissions.js'; import { userPaths } from './users.js'; +import { aiServicesPaths } from './aiServices.js'; export const analyticsPaths = { '/v1/admin/analytics': { @@ -94,6 +95,7 @@ export const openaiDocumentation = createDocument({ { name: 'OpenAlex', description: 'OpenAlex related operations' }, { name: 'Authors', description: 'Authors-related operations' }, { name: 'Search', description: 'Elastic search api operations' }, + { name: 'AI Services', description: 'AI-powered research tools and services' }, ], components: { securitySchemes: { @@ -137,6 +139,7 @@ export const openaiDocumentation = createDocument({ ...journalPaths, ...adminNodesPaths, ...metricsPaths, + ...aiServicesPaths, }, }); diff --git a/desci-server/src/services/externalApi/RefereeRecommenderService.ts b/desci-server/src/services/externalApi/RefereeRecommenderService.ts index 222b67450..8a87cb03a 100644 --- a/desci-server/src/services/externalApi/RefereeRecommenderService.ts +++ b/desci-server/src/services/externalApi/RefereeRecommenderService.ts @@ -712,5 +712,5 @@ export const RefereeRecommenderService = { handleProcessingFailure, cleanupExpiredSessions, prepareFormattedFileName, - prepareCacheKey: prepareSessionCacheKey, + prepareSessionCacheKey, }; diff --git a/desci-server/src/services/externalApi/RefereeRecommenderSqsHandler.ts b/desci-server/src/services/externalApi/RefereeRecommenderSqsHandler.ts index 33b949a74..949050ccc 100644 --- a/desci-server/src/services/externalApi/RefereeRecommenderSqsHandler.ts +++ b/desci-server/src/services/externalApi/RefereeRecommenderSqsHandler.ts @@ -82,6 +82,8 @@ export class RefereeRecommenderSqsHandler { debugger; const fileUrl = eventData.data.file_url; + const hashedFileUrl = RefereeRecommenderService.prepareSessionCacheKey(fileUrl); + eventData.fileUrlHashed = hashedFileUrl; // Attach hashed version of file url // Look up user sessions from Redis using filename const sessionsResult = await RefereeRecommenderService.getSessionsByFileUrl(fileUrl); @@ -129,6 +131,8 @@ export class RefereeRecommenderSqsHandler { originalFileName: session.originalFileName, status: 'completed', timestamp: new Date().toISOString(), + fileUrlHashed: eventData.fileUrlHashed, + paperTitle: eventData.data?.paper_title, }; break; @@ -140,6 +144,8 @@ export class RefereeRecommenderSqsHandler { status: 'failed', error: eventData.data?.error || 'Processing failed', timestamp: new Date().toISOString(), + fileUrlHashed: eventData.fileUrlHashed, + paperTitle: eventData.data?.paper_title, }; break; @@ -150,6 +156,8 @@ export class RefereeRecommenderSqsHandler { originalFileName: session.originalFileName, status: 'started', timestamp: new Date().toISOString(), + fileUrlHashed: eventData.fileUrlHashed, + paperTitle: eventData.data?.paper_title, }; break; @@ -162,6 +170,8 @@ export class RefereeRecommenderSqsHandler { progress: eventData.data?.progress_percent || 0, message: eventData.data?.message || '', timestamp: new Date().toISOString(), + fileUrlHashed: eventData.fileUrlHashed, + paperTitle: eventData.data?.paper_title, }; break; diff --git a/desci-server/src/services/sqs/SqsMessageTypes.ts b/desci-server/src/services/sqs/SqsMessageTypes.ts index 015d51c41..3d5d8fc6e 100644 --- a/desci-server/src/services/sqs/SqsMessageTypes.ts +++ b/desci-server/src/services/sqs/SqsMessageTypes.ts @@ -30,7 +30,8 @@ export interface ExternalApiSqsMessage extends BaseSqsMessage { file_name: string; // e.g., "referee_rec_v0.1.3_{original_file.pdf}" // Move to hashes later for security. file_hash?: string; // SHA-256 hash of the file content file_url: string; // Source URL - data?: any; + data?: ExternalApiSqsMessageData | any; + fileUrlHashed: string; } export type ExternalApiSqsMessageData = { From 74c1505225d2850c0a4f2e28e20cebdd2e30bb9c Mon Sep 17 00:00:00 2001 From: kadami <86646883+kadamidev@users.noreply.github.com> Date: Thu, 21 Aug 2025 21:15:49 +1000 Subject: [PATCH 09/10] fix nasty double hash issue, update getRes controller to check on completed results and not sessions --- .../RefereeRecommender/getResults.ts | 43 +++++++++++-------- .../triggerRecommendation.ts | 6 ++- .../externalApi/RefereeRecommenderService.ts | 17 ++++---- 3 files changed, 38 insertions(+), 28 deletions(-) diff --git a/desci-server/src/controllers/externalApi/RefereeRecommender/getResults.ts b/desci-server/src/controllers/externalApi/RefereeRecommender/getResults.ts index 6b97d201a..2ffad8007 100644 --- a/desci-server/src/controllers/externalApi/RefereeRecommender/getResults.ts +++ b/desci-server/src/controllers/externalApi/RefereeRecommender/getResults.ts @@ -1,6 +1,8 @@ +import { ExternalApi } from '@prisma/client'; import { Response } from 'express'; import { z } from 'zod'; +import { prisma } from '../../../client.js'; import { sendSuccess, sendError } from '../../../core/api.js'; import { AuthenticatedRequest } from '../../../core/types.js'; import { logger as parentLogger } from '../../../logger.js'; @@ -29,32 +31,33 @@ export const getResults = async (req: AuthenticatedRequest, res: Response) => { 'Fetching referee recommendation results', ); - // Verify user has access to this filename by checking session - const sessionResult = await RefereeRecommenderService.getSession(UploadedFileName, user.id); - if (sessionResult.isErr()) { - logger.warn( - { - userId: user.id, - uploadedFileName: UploadedFileName, + // Check ExternalApiUsage table for completed results + const existingResult = await prisma.externalApiUsage.findFirst({ + where: { + userId: user.id, + apiType: ExternalApi.REFEREE_FINDER, + queryingData: { + path: ['fileName'], + equals: UploadedFileName, }, - 'Session not found or expired', - ); - return sendError(res, 'Results not found or access denied', 404); - } + }, + orderBy: { + createdAt: 'desc', // Get the most recent result + }, + }); - const session = sessionResult.value; - if (session.userId !== user.id) { + if (!existingResult) { logger.warn( { userId: user.id, - sessionUserId: session.userId, uploadedFileName: UploadedFileName, }, - 'User does not have access to this session', + 'No results found for this filename and user', ); - return sendError(res, 'Results not found or access denied', 404); + return sendError(res, 'Results not found', 404); } + // User has access to this filename, now fetch actual results from lambda API const result = await RefereeRecommenderService.getRefereeResults(UploadedFileName); if (result.isErr()) { @@ -63,8 +66,9 @@ export const getResults = async (req: AuthenticatedRequest, res: Response) => { error: result.error, userId: user.id, uploadedFileName: UploadedFileName, + resultId: existingResult.id, }, - 'Failed to fetch referee results', + 'Failed to fetch referee results from lambda API', ); return sendError(res, result.error.message, 500); } @@ -75,9 +79,10 @@ export const getResults = async (req: AuthenticatedRequest, res: Response) => { { userId: user.id, uploadedFileName: UploadedFileName, - status: responseData.status, + resultId: existingResult.id, + createdAt: existingResult.createdAt, }, - 'Successfully fetched referee recommendation results', + 'Successfully fetched referee recommendation results from database', ); return sendSuccess(res, responseData); diff --git a/desci-server/src/controllers/externalApi/RefereeRecommender/triggerRecommendation.ts b/desci-server/src/controllers/externalApi/RefereeRecommender/triggerRecommendation.ts index a08c14647..134526baa 100644 --- a/desci-server/src/controllers/externalApi/RefereeRecommender/triggerRecommendation.ts +++ b/desci-server/src/controllers/externalApi/RefereeRecommender/triggerRecommendation.ts @@ -115,9 +115,13 @@ export const triggerRecommendation = async (req: AuthenticatedRequest, res: Resp const formattedResponse = _.omit(responseData, 'execution_arn'); + // Check if results are already available based on info message + const isResultReady = responseData.info?.toLowerCase().includes('already exists') && + responseData.info?.toLowerCase().includes('ready to be polled'); + return sendSuccess(res, { ...formattedResponse, - cached: false, + cached: isResultReady, }); } catch (error) { if (error instanceof z.ZodError) { diff --git a/desci-server/src/services/externalApi/RefereeRecommenderService.ts b/desci-server/src/services/externalApi/RefereeRecommenderService.ts index 8a87cb03a..8640c0a74 100644 --- a/desci-server/src/services/externalApi/RefereeRecommenderService.ts +++ b/desci-server/src/services/externalApi/RefereeRecommenderService.ts @@ -162,7 +162,7 @@ async function generatePresignedUploadUrl(request: PresignedUrlRequest): Promise const hashedFileUrl = prepareSessionCacheKey(download_url); // Store session in Redis using the hashed download URL - const storeResult = await storeSession(hashedFileUrl, { + const storeResult = await storeSession(download_url, { userId: request.userId, hashedFileUrl, createdAt: Date.now(), @@ -172,7 +172,7 @@ async function generatePresignedUploadUrl(request: PresignedUrlRequest): Promise return err(storeResult.error); } - await markSessionAsActive(request.userId, hashedFileUrl); + await markSessionAsActive(request.userId, download_url); logger.info( { @@ -207,6 +207,7 @@ function generateFileName(originalFileName: string): string { async function storeSession(fileUrl: string, session: RefereeRecommenderSession): Promise> { try { const hashedFileUrl = prepareSessionCacheKey(fileUrl); + debugger; const userCacheKey = `referee-recommender-session:${session.userId}:${hashedFileUrl}`; await setToCache(userCacheKey, session, SESSION_TTL_SECONDS); @@ -242,20 +243,20 @@ async function storeSession(fileUrl: string, session: RefereeRecommenderSession) } } -async function getSession(fileName: string, userId: number): Promise> { +async function getSession(fileUrlHash: string, userId: number): Promise> { try { - const cacheKey = `referee-recommender-session:${userId}:${fileName}`; + const cacheKey = `referee-recommender-session:${userId}:${fileUrlHash}`; const session = await getFromCache(cacheKey); if (session) { - logger.debug({ fileName, userId: session.userId }, 'Found referee recommender session'); + logger.debug({ fileUrlHash, userId: session.userId }, 'Found referee recommender session'); return ok(session); } - logger.debug({ fileName, userId }, 'Session not found in Redis'); + logger.debug({ fileUrlHash, userId }, 'Session not found in Redis'); return err(new Error('Session not found')); } catch (error) { - logger.error({ error, fileName, userId }, 'Failed to get session from Redis'); + logger.error({ error, fileUrlHash, userId }, 'Failed to get session from Redis'); return err(error instanceof Error ? error : new Error('Failed to get session from Redis')); } } @@ -263,7 +264,7 @@ async function getSession(fileName: string, userId: number): Promise> { try { const hashedFileUrl = prepareSessionCacheKey(fileUrl); - const filenameCacheKey = `referee-recommender-file-url:${fileUrl}`; + const filenameCacheKey = `referee-recommender-file-url:${hashedFileUrl}`; const sessions = await getFromCache(filenameCacheKey); if (sessions && sessions.length > 0) { From 687ef41942e7293c0e3e730ba8674dd1e1aa5f0c Mon Sep 17 00:00:00 2001 From: kadami <86646883+kadamidev@users.noreply.github.com> Date: Fri, 22 Aug 2025 07:43:18 +1000 Subject: [PATCH 10/10] fixes --- .../triggerRecommendation.ts | 5 +++-- .../externalApi/RefereeRecommenderService.ts | 2 +- .../RefereeRecommenderSqsHandler.ts | 19 +++++++++++-------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/desci-server/src/controllers/externalApi/RefereeRecommender/triggerRecommendation.ts b/desci-server/src/controllers/externalApi/RefereeRecommender/triggerRecommendation.ts index 134526baa..f5a0d96a5 100644 --- a/desci-server/src/controllers/externalApi/RefereeRecommender/triggerRecommendation.ts +++ b/desci-server/src/controllers/externalApi/RefereeRecommender/triggerRecommendation.ts @@ -116,8 +116,9 @@ export const triggerRecommendation = async (req: AuthenticatedRequest, res: Resp const formattedResponse = _.omit(responseData, 'execution_arn'); // Check if results are already available based on info message - const isResultReady = responseData.info?.toLowerCase().includes('already exists') && - responseData.info?.toLowerCase().includes('ready to be polled'); + const isResultReady = + responseData.info?.toLowerCase().includes('already exists') && + responseData.info?.toLowerCase().includes('ready to be polled'); return sendSuccess(res, { ...formattedResponse, diff --git a/desci-server/src/services/externalApi/RefereeRecommenderService.ts b/desci-server/src/services/externalApi/RefereeRecommenderService.ts index 8640c0a74..11719186a 100644 --- a/desci-server/src/services/externalApi/RefereeRecommenderService.ts +++ b/desci-server/src/services/externalApi/RefereeRecommenderService.ts @@ -207,7 +207,7 @@ function generateFileName(originalFileName: string): string { async function storeSession(fileUrl: string, session: RefereeRecommenderSession): Promise> { try { const hashedFileUrl = prepareSessionCacheKey(fileUrl); - debugger; + const userCacheKey = `referee-recommender-session:${session.userId}:${hashedFileUrl}`; await setToCache(userCacheKey, session, SESSION_TTL_SECONDS); diff --git a/desci-server/src/services/externalApi/RefereeRecommenderSqsHandler.ts b/desci-server/src/services/externalApi/RefereeRecommenderSqsHandler.ts index 949050ccc..a83068a8f 100644 --- a/desci-server/src/services/externalApi/RefereeRecommenderSqsHandler.ts +++ b/desci-server/src/services/externalApi/RefereeRecommenderSqsHandler.ts @@ -80,10 +80,8 @@ export class RefereeRecommenderSqsHandler { '[SQS] Processing referee recommender event', ); - debugger; + // debugger; const fileUrl = eventData.data.file_url; - const hashedFileUrl = RefereeRecommenderService.prepareSessionCacheKey(fileUrl); - eventData.fileUrlHashed = hashedFileUrl; // Attach hashed version of file url // Look up user sessions from Redis using filename const sessionsResult = await RefereeRecommenderService.getSessionsByFileUrl(fileUrl); @@ -123,6 +121,11 @@ export class RefereeRecommenderSqsHandler { try { let eventType: WebSocketEventType; let eventPayload: any; + + // Compute fileUrlHashed using our consistent hashing method + // This ensures it matches what the frontend computed + const fileUrlHashed = RefereeRecommenderService.prepareSessionCacheKey(eventData.data.file_url); + switch (eventData.eventType) { case 'PROCESSING_COMPLETED': eventType = WebSocketEventType.REFEREE_REC_PROCESSING_COMPLETED; @@ -131,7 +134,7 @@ export class RefereeRecommenderSqsHandler { originalFileName: session.originalFileName, status: 'completed', timestamp: new Date().toISOString(), - fileUrlHashed: eventData.fileUrlHashed, + fileUrlHashed, paperTitle: eventData.data?.paper_title, }; break; @@ -144,7 +147,7 @@ export class RefereeRecommenderSqsHandler { status: 'failed', error: eventData.data?.error || 'Processing failed', timestamp: new Date().toISOString(), - fileUrlHashed: eventData.fileUrlHashed, + fileUrlHashed, paperTitle: eventData.data?.paper_title, }; break; @@ -156,7 +159,7 @@ export class RefereeRecommenderSqsHandler { originalFileName: session.originalFileName, status: 'started', timestamp: new Date().toISOString(), - fileUrlHashed: eventData.fileUrlHashed, + fileUrlHashed, paperTitle: eventData.data?.paper_title, }; break; @@ -167,10 +170,10 @@ export class RefereeRecommenderSqsHandler { fileName: eventData.file_name, originalFileName: session.originalFileName, status: 'processing', - progress: eventData.data?.progress_percent || 0, + progress_percent: eventData.data?.progress_percent || 0, // Keep original field name message: eventData.data?.message || '', timestamp: new Date().toISOString(), - fileUrlHashed: eventData.fileUrlHashed, + fileUrlHashed, paperTitle: eventData.data?.paper_title, }; break;