[QUO-898] document metadata support/validation (#7)

* document metadata support/validation

* verbose + actionable error msgs

* code coverage tests

* bug fix on listing logs

* test fixes

* edge case with empty response

* update v num in package.json

* update example logs

Author: crekhari

examples/example_logs.ts

@@ -16,15 +16,18 @@ async function main() {
     console.log("Logger initialized")
 
     // mock retrieved documents
-    const retrieved_documents = [{"page_content": "Sample document"}]
-    const documents = retrieved_documents.map(doc => doc["page_content"])
+    const retrieved_documents = [
+        "Sample document 1",
+        {"page_content": "Sample document 2", "metadata": {"source": "website.com"}},
+        {"page_content": "Sample document 3"}
+    ]
 
     console.log("Preparing to log with quotient_logger")
     try {
         const response = await quotient_logger.log({
             user_query: "How do I cook a goose?",
             model_output: "The capital of France is Paris",
-            documents: documents,
+            documents: retrieved_documents,
             message_history: [
                 {"role": "system", "content": "You are an expert on geography."},
                 {"role": "user", "content": "What is the capital of France?"},

package.json

@@ -1,6 +1,6 @@
 {
   "name": "quotientai",
-  "version": "0.0.3",
+  "version": "0.0.4",
   "description": "TypeScript client for QuotientAI API",
   "main": "dist/quotientai/index.js",
   "types": "dist/quotientai/index.d.ts",

quotientai/exceptions.ts

@@ -1,4 +1,3 @@
-
 import axios, { AxiosError, AxiosRequestConfig, AxiosResponse, InternalAxiosRequestConfig } from 'axios';
 
 export class QuotientAIError extends Error {
@@ -8,6 +7,13 @@ export class QuotientAIError extends Error {
     }
 }
 
+export class ValidationError extends QuotientAIError {
+    constructor(message: string) {
+        super(message);
+        this.name = 'ValidationError';
+    }
+}
+
 export class APIError extends QuotientAIError {
     request: AxiosRequestConfig;
     body: any;

quotientai/logger.ts

@@ -1,4 +1,5 @@
-import { LogEntry, LoggerConfig } from './types';
+import { LogEntry, LoggerConfig, LogDocument } from './types';
+import { ValidationError } from './exceptions';
 
 interface LogsResource {
   create(params: LogEntry): Promise<any>;
@@ -41,6 +42,67 @@ export class QuotientLogger {
     return Math.random() < this.sampleRate;
   }
 
+  // Type guard function to check if an object is a valid LogDocument
+  private isValidLogDocument(obj: any): { valid: boolean; error?: string } {
+    try {
+      // Check if it has the required page_content property
+      if (!('page_content' in obj)) {
+        return { 
+          valid: false, 
+          error: "Missing required 'page_content' property" 
+        };
+      }
+      
+      // Check if page_content is a string
+      if (typeof obj.page_content !== 'string') {
+        return { 
+          valid: false, 
+          error: `The 'page_content' property must be a string, found ${typeof obj.page_content}` 
+        };
+      }
+      
+      // If metadata exists, check if it's an object
+      if ('metadata' in obj && obj.metadata !== null && typeof obj.metadata !== 'object') {
+        return { 
+          valid: false, 
+          error: `The 'metadata' property must be an object, found ${typeof obj.metadata}` 
+        };
+      }
+      
+      return { valid: true };
+    } catch (error) {
+      return { valid: false, error: "Unexpected error validating document" };
+    }
+  }
+
+  // Validate document format
+  private validateDocuments(documents: (string | LogDocument)[]): void {
+    if (!documents || documents.length === 0) {
+      return;
+    }
+
+    for (let i = 0; i < documents.length; i++) {
+      const doc = documents[i];
+      if (typeof doc === 'string') {
+        continue;
+      } else if (typeof doc === 'object' && doc !== null) {
+        const validation = this.isValidLogDocument(doc);
+        if (!validation.valid) {
+          throw new ValidationError(
+            `Invalid document format at index ${i}: ${validation.error}. ` +
+            "Documents must be either strings or JSON objects with a 'page_content' string property and an optional 'metadata' object. " +
+            "To fix this, ensure each document follows the format: { page_content: 'your text content', metadata?: { key: 'value' } }"
+          );
+        }
+      } else {
+        throw new ValidationError(
+          `Invalid document type at index ${i}. Found ${typeof doc}, but documents must be either strings or JSON objects with a 'page_content' property. ` +
+          "To fix this, provide documents as either simple strings or properly formatted objects: { page_content: 'your text content' }"
+        );
+      }
+    }
+  }
+
   // log a message
   // params: Omit<LogEntry, 'app_name' | 'environment'>
   async log(params: Omit<LogEntry, 'app_name' | 'environment'>): Promise<any> {
@@ -52,6 +114,11 @@ export class QuotientLogger {
       throw new Error('Logger is not properly configured. app_name and environment must be set.');
     }
 
+    // Validate documents format
+    if (params.documents) {
+      this.validateDocuments(params.documents);
+    }
+
     // Merge default tags with any tags provided at log time
     const mergedTags = { ...this.tags, ...(params.tags || {}) };
 

quotientai/resources/logs.ts

@@ -1,4 +1,5 @@
 import { BaseQuotientClient } from '../client';
+import { LogDocument } from '../types';
 
 interface LogResponse {
     id: string;
@@ -8,7 +9,7 @@ interface LogResponse {
     inconsistency_detection: boolean;
     user_query: string;
     model_output: string;
-    documents: string[];
+    documents: (string | LogDocument)[];
     message_history: any[] | null;
     instructions: string[] | null;
     tags: Record<string, any>;
@@ -26,7 +27,7 @@ interface CreateLogParams {
     inconsistency_detection: boolean;
     user_query: string;
     model_output: string;
-    documents: string[];
+    documents: (string | LogDocument)[];
     message_history?: any[] | null;
     instructions?: string[] | null;
     tags?: Record<string, any>;
@@ -50,7 +51,7 @@ export class Log {
     inconsistency_detection: boolean;
     user_query: string;
     model_output: string;
-    documents: string[];
+    documents: (string | LogDocument)[];
     message_history: any[] | null;
     instructions: string[] | null;
     tags: Record<string, any>;
@@ -109,7 +110,15 @@ export class LogsResource {
         if (params.offset !== undefined) queryParams.offset = params.offset;
 
         try {
-            const response = await this.client.get('/logs', { params: queryParams }) as LogsResponse;
+            const response = await this.client.get('/logs', queryParams) as LogsResponse;
+            
+            // Check if response has logs property and it's an array
+            if (!response || !response.logs || !Array.isArray(response.logs)) {
+                console.warn('No logs found. Please check your query parameters and try again.');
+                return [];
+            }
+
+            // Map the logs to Log objects
             return response.logs.map(logData => new Log(logData));
         } catch (error) {
             console.error('Error listing logs:', error);

quotientai/types.ts

@@ -75,12 +75,17 @@ export interface RunResult {
   expected: string;
 }
 
+export interface LogDocument {
+  page_content: string;
+  metadata?: Record<string, any>;
+}
+
 export interface LogEntry {
   app_name: string;
   environment: string;
   user_query: string;
   model_output: string;
-  documents: string[];
+  documents: (string | LogDocument)[];
   message_history?: Array<Record<string, any>> | null; 
   instructions?: string[] | null;
   tags?: Record<string, any>;