Ref API changes

.env.example

@@ -203,8 +203,9 @@ MIXPANEL_TOKEN=
 
 # Referee Finder
 ML_REFEREE_GET_PRESIGNED_URL=
-ML_REFEREE_TRIGGER_CID=
+ML_REFEREE_TRIGGER_START=
 ML_REFEREE_FINDER_RESULT=
+ML_REFEREE_FILE_PREFIX=
 
 # Temp var for metering comms between nextjs api backend/nodes backend
 INTERNAL_SERVICE_SECRET=psssssssssst

desci-server/kubernetes/deployment_dev.yaml

@@ -114,8 +114,9 @@ spec:
           export AWS_SQS_DATA_MIGRATION_QUEUE_URL="{{ .Data.AWS_SQS_DATA_MIGRATION_QUEUE_URL }}"
           export AWS_SQS_ML_TOOL_QUEUE_URL="{{ .Data.AWS_SQS_ML_TOOL_QUEUE_URL }}"
           export ML_REFEREE_GET_PRESIGNED_URL="{{ .Data.ML_REFEREE_GET_PRESIGNED_URL }}"
-          export ML_REFEREE_TRIGGER_CID="{{ .Data.ML_REFEREE_TRIGGER_CID }}"
+          export ML_REFEREE_TRIGGER_START="{{ .Data.ML_REFEREE_TRIGGER_START }}"
           export ML_REFEREE_FINDER_RESULT="{{ .Data.ML_REFEREE_FINDER_RESULT }}"
+          export ML_REFEREE_FILE_PREFIX="{{ .Data.ML_REFEREE_FILE_PREFIX }}"
           export ENABLE_GUEST_MODE="{{ .Data.ENABLE_GUEST_MODE }}"
           export MIXPANEL_TOKEN="{{ .Data.MIXPANEL_TOKEN }}"
           export SCIWEAVE_GOOGLE_CLIENT_ID="{{ .Data.SCIWEAVE_GOOGLE_CLIENT_ID }}"

desci-server/kubernetes/deployment_prod.yaml

@@ -113,8 +113,9 @@ spec:
           export AWS_SQS_DATA_MIGRATION_QUEUE_URL="{{ .Data.AWS_SQS_DATA_MIGRATION_QUEUE_URL }}"
           export AWS_SQS_ML_TOOL_QUEUE_URL="{{ .Data.AWS_SQS_ML_TOOL_QUEUE_URL }}"
           export ML_REFEREE_GET_PRESIGNED_URL="{{ .Data.ML_REFEREE_GET_PRESIGNED_URL }}"
-          export ML_REFEREE_TRIGGER_CID="{{ .Data.ML_REFEREE_TRIGGER_CID }}"
+          export ML_REFEREE_TRIGGER_START="{{ .Data.ML_REFEREE_TRIGGER_START }}"
           export ML_REFEREE_FINDER_RESULT="{{ .Data.ML_REFEREE_FINDER_RESULT }}"
+          export ML_REFEREE_FILE_PREFIX="{{ .Data.ML_REFEREE_FILE_PREFIX }}"
           export ENABLE_GUEST_MODE="{{ .Data.ENABLE_GUEST_MODE }}"
           export MIXPANEL_TOKEN="{{ .Data.MIXPANEL_TOKEN }}"
           export SCIWEAVE_GOOGLE_CLIENT_ID="{{ .Data.SCIWEAVE_GOOGLE_CLIENT_ID }}"

desci-server/kubernetes/deployment_staging.yaml

@@ -120,8 +120,9 @@ spec:
           export AWS_SQS_DATA_MIGRATION_QUEUE_URL="{{ .Data.AWS_SQS_DATA_MIGRATION_QUEUE_URL }}"
           export AWS_SQS_ML_TOOL_QUEUE_URL="{{ .Data.AWS_SQS_ML_TOOL_QUEUE_URL }}"
           export ML_REFEREE_GET_PRESIGNED_URL="{{ .Data.ML_REFEREE_GET_PRESIGNED_URL }}"
-          export ML_REFEREE_TRIGGER_CID="{{ .Data.ML_REFEREE_TRIGGER_CID }}"
+          export ML_REFEREE_TRIGGER_START="{{ .Data.ML_REFEREE_TRIGGER_START }}"
           export ML_REFEREE_FINDER_RESULT="{{ .Data.ML_REFEREE_FINDER_RESULT }}"
+          export ML_REFEREE_FILE_PREFIX="{{ .Data.ML_REFEREE_FILE_PREFIX }}"
           export ENABLE_GUEST_MODE="{{ .Data.ENABLE_GUEST_MODE }}"
           export MIXPANEL_TOKEN="{{ .Data.MIXPANEL_TOKEN }}"
           export SCIWEAVE_GOOGLE_CLIENT_ID="{{ .Data.SCIWEAVE_GOOGLE_CLIENT_ID }}"

desci-server/prisma/migrations/20250818192731_add_querying_data_external_api_table/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "ExternalApiUsage" ADD COLUMN     "queryingData" JSONB;

desci-server/prisma/migrations/20250818192759_add_gin_index_external_api_querying_data/migration.sql

@@ -0,0 +1,6 @@
+-- CreateIndex: Add GIN index for ExternalApiUsage.queryingData field
+-- This allows efficient JSON containment queries on the queryingData field
+
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "idx_external_api_usage_querying_data_gin" 
+ON "ExternalApiUsage" USING GIN ("queryingData");
+

desci-server/prisma/migrations/20250818194118_gin_index_ext_api_querying_data/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX "idx_external_api_usage_querying_data_gin";

desci-server/prisma/schema.prisma

@@ -1435,12 +1435,13 @@ model JournalFormResponse {
 }
 
 model ExternalApiUsage {
-  id        Int         @id @default(autoincrement())
-  userId    Int
-  apiType   ExternalApi
-  createdAt DateTime    @default(now())
-  updatedAt DateTime    @updatedAt
-  data      Json?
+  id           Int         @id @default(autoincrement())
+  userId       Int
+  apiType      ExternalApi
+  createdAt    DateTime    @default(now())
+  updatedAt    DateTime    @updatedAt
+  data         Json? // Can dump additional metadata here
+  queryingData Json? // GIN indexed, important data thats used for querying goes here, avoid dumping random metadata.
 
   // Relationships
   user User @relation(fields: [userId], references: [id])

desci-server/src/controllers/externalApi/RefereeRecommender/getResults.ts

@@ -1,6 +1,8 @@
+import { ExternalApi } from '@prisma/client';
 import { Response } from 'express';
 import { z } from 'zod';
 
+import { prisma } from '../../../client.js';
 import { sendSuccess, sendError } from '../../../core/api.js';
 import { AuthenticatedRequest } from '../../../core/types.js';
 import { logger as parentLogger } from '../../../logger.js';
@@ -29,32 +31,33 @@ export const getResults = async (req: AuthenticatedRequest, res: Response) => {
       'Fetching referee recommendation results',
     );
 
-    // Verify user has access to this filename by checking session
-    const sessionResult = await RefereeRecommenderService.getSession(UploadedFileName, user.id);
-    if (sessionResult.isErr()) {
-      logger.warn(
-        {
-          userId: user.id,
-          uploadedFileName: UploadedFileName,
+    // Check ExternalApiUsage table for completed results
+    const existingResult = await prisma.externalApiUsage.findFirst({
+      where: {
+        userId: user.id,
+        apiType: ExternalApi.REFEREE_FINDER,
+        queryingData: {
+          path: ['fileName'],
+          equals: UploadedFileName,
         },
-        'Session not found or expired',
-      );
-      return sendError(res, 'Results not found or access denied', 404);
-    }
+      },
+      orderBy: {
+        createdAt: 'desc', // Get the most recent result
+      },
+    });
 
-    const session = sessionResult.value;
-    if (session.userId !== user.id) {
+    if (!existingResult) {
       logger.warn(
         {
           userId: user.id,
-          sessionUserId: session.userId,
           uploadedFileName: UploadedFileName,
         },
-        'User does not have access to this session',
+        'No results found for this filename and user',
       );
-      return sendError(res, 'Results not found or access denied', 404);
+      return sendError(res, 'Results not found', 404);
     }
 
+    // User has access to this filename, now fetch actual results from lambda API
     const result = await RefereeRecommenderService.getRefereeResults(UploadedFileName);
 
     if (result.isErr()) {
@@ -63,8 +66,9 @@ export const getResults = async (req: AuthenticatedRequest, res: Response) => {
           error: result.error,
           userId: user.id,
           uploadedFileName: UploadedFileName,
+          resultId: existingResult.id,
         },
-        'Failed to fetch referee results',
+        'Failed to fetch referee results from lambda API',
       );
       return sendError(res, result.error.message, 500);
     }
@@ -75,9 +79,10 @@ export const getResults = async (req: AuthenticatedRequest, res: Response) => {
       {
         userId: user.id,
         uploadedFileName: UploadedFileName,
-        status: responseData.status,
+        resultId: existingResult.id,
+        createdAt: existingResult.createdAt,
       },
-      'Successfully fetched referee recommendation results',
+      'Successfully fetched referee recommendation results from database',
     );
 
     return sendSuccess(res, responseData);

desci-server/src/controllers/externalApi/RefereeRecommender/issuePresignedUrl.ts

@@ -1,6 +1,8 @@
+import { ExternalApi } from '@prisma/client';
 import { Response } from 'express';
 import { z } from 'zod';
 
+import { prisma } from '../../../client.js';
 import { sendSuccess, sendError } from '../../../core/api.js';
 import { AuthenticatedRequest } from '../../../core/types.js';
 import { logger as parentLogger } from '../../../logger.js';
@@ -10,6 +12,7 @@ const logger = parentLogger.child({ module: 'RefereeRecommender::PresignedUrlCon
 
 const generatePresignedUrlSchema = z.object({
   body: z.object({
+    fileHash: z.string().min(1, 'fileHash is required'),
     fileName: z.string().min(1, 'fileName is required'),
   }),
 });
@@ -24,17 +27,51 @@ export const generatePresignedUrl = async (req: AuthenticatedRequest, res: Respo
     }
 
     const { body } = generatePresignedUrlSchema.parse(req);
-    const { fileName } = body;
+    const { fileHash, fileName } = body;
 
     logger.info(
       {
         userId: user.id,
+        fileHash,
         fileName,
       },
-      'Generating presigned URL for referee recommender',
+      'Checking cache and potentially generating presigned URL for referee recommender',
     );
 
-    // Validate file extension (assuming PDFs for now)
+    // First, check if we have cached results for this fileHash
+    const cachedResult = await prisma.externalApiUsage.findFirst({
+      where: {
+        userId: user.id,
+        apiType: ExternalApi.REFEREE_FINDER,
+        queryingData: {
+          path: ['fileHash'],
+          equals: fileHash,
+        },
+      },
+      orderBy: {
+        createdAt: 'desc', // Get the most recent result
+      },
+    });
+
+    if (cachedResult) {
+      logger.info(
+        {
+          userId: user.id,
+          fileHash,
+          cachedResultId: cachedResult.id,
+        },
+        'Found cached results for fileHash',
+      );
+
+      return sendSuccess(res, {
+        cached: true,
+        message: 'Results for this file are already available',
+        resultKey: RefereeRecommenderService.prepareFormattedFileName(fileHash),
+      });
+    }
+
+    // No cached results found, proceed with generating presigned URL
+    // Validate file extension
     if (!fileName.toLowerCase().endsWith('.pdf')) {
       return sendError(res, 'Only PDF files are supported', 400);
     }
@@ -49,7 +86,7 @@ export const generatePresignedUrl = async (req: AuthenticatedRequest, res: Respo
       return sendError(res, 'Failed to generate presigned URL', 500);
     }
 
-    const { presignedUrl, fileName: generatedFileName } = result.value;
+    const { presignedUrl, downloadUrl, fileName: generatedFileName } = result.value;
 
     logger.info(
       {
@@ -60,9 +97,11 @@ export const generatePresignedUrl = async (req: AuthenticatedRequest, res: Respo
     );
 
     return sendSuccess(res, {
+      cached: false,
       presignedUrl,
+      downloadUrl,
       fileName: generatedFileName,
-      expiresIn: 3600, // 1 hour
+      fileHash,
     });
   } catch (error) {
     if (error instanceof z.ZodError) {

desci-server/src/controllers/externalApi/RefereeRecommender/triggerRecommendation.ts

@@ -1,6 +1,9 @@
+import { ExternalApi } from '@prisma/client';
 import { Response } from 'express';
+import _ from 'lodash';
 import { z } from 'zod';
 
+import { prisma } from '../../../client.js';
 import { sendSuccess, sendError } from '../../../core/api.js';
 import { AuthenticatedRequest } from '../../../core/types.js';
 import { logger as parentLogger } from '../../../logger.js';
@@ -19,18 +22,69 @@ export const triggerRecommendation = async (req: AuthenticatedRequest, res: Resp
     }
 
     const { body } = triggerRefereeRecommendationSchema.parse(req);
+    const { fileUrl, fileHash } = body;
 
     logger.info(
       {
         userId: user.id,
-        cid: body.cid,
-        external: body.external,
+        fileUrl,
+        fileHash,
       },
-      'Triggering referee recommendation',
+      'Checking cache and potentially triggering referee recommendation',
     );
 
+    // Check if we have cached results for this fileHash (if provided)
+    let cachedResult = null;
+
+    if (fileHash) {
+      cachedResult = await prisma.externalApiUsage.findFirst({
+        where: {
+          userId: user.id,
+          apiType: ExternalApi.REFEREE_FINDER,
+          queryingData: {
+            path: ['fileHash'],
+            equals: fileHash,
+          },
+        },
+        orderBy: {
+          createdAt: 'desc', // Get the most recent result
+        },
+      });
+
+      if (cachedResult) {
+        logger.info(
+          {
+            userId: user.id,
+            fileHash,
+            cachedResultId: cachedResult.id,
+          },
+          'Found cached results for fileHash, returning cached data',
+        );
+
+        return sendSuccess(res, {
+          cached: true,
+          message: 'Results for this file are already available',
+          createdAt: cachedResult.createdAt,
+          resultKey: RefereeRecommenderService.prepareFormattedFileName(fileHash),
+        });
+      }
+    }
+
     const result = await RefereeRecommenderService.triggerRefereeRecommendation(
-      body as typeof body & { cid: string },
+      {
+        // hash_value,
+        // hash_verified,
+        file_url: body.fileUrl,
+        top_n_closely_matching: body.top_n_closely_matching,
+        number_referees: body.number_referees,
+        force_run: body.force_run,
+        classify: body.classify,
+        coi_filter: body.coi_filter,
+        meta_data_only: body.meta_data_only,
+        exclude_fields: body.exclude_fields,
+        exclude_works: body.exclude_works,
+        exclude_authors: body.exclude_authors,
+      },
       user.id,
     );
 
@@ -39,7 +93,6 @@ export const triggerRecommendation = async (req: AuthenticatedRequest, res: Resp
         {
           error: result.error,
           userId: user.id,
-          cid: body.cid,
         },
         'Failed to trigger referee recommendation',
       );
@@ -54,13 +107,23 @@ export const triggerRecommendation = async (req: AuthenticatedRequest, res: Resp
     logger.info(
       {
         userId: user.id,
-        uploadedFileName: responseData.uploaded_file_name,
+        uploadedFileName: responseData.UploadedFileName,
         info: responseData.info,
       },
       'Successfully triggered referee recommendation',
     );
 
-    return sendSuccess(res, responseData);
+    const formattedResponse = _.omit(responseData, 'execution_arn');
+
+    // Check if results are already available based on info message
+    const isResultReady =
+      responseData.info?.toLowerCase().includes('already exists') &&
+      responseData.info?.toLowerCase().includes('ready to be polled');
+
+    return sendSuccess(res, {
+      ...formattedResponse,
+      cached: isResultReady,
+    });
   } catch (error) {
     if (error instanceof z.ZodError) {
       logger.warn({ error: error.errors }, 'Invalid request parameters');