diff --git a/tests/integration/client_side_evals.ts b/tests/integration/client_side_evals.ts new file mode 100644 index 0000000..4a324b4 --- /dev/null +++ b/tests/integration/client_side_evals.ts @@ -0,0 +1,81 @@ +import { HoneyHiveTracer, HoneyHive } from "honeyhive"; +import assert from "assert"; +import { Operator } from "honeyhive/models/components/index.js"; + +async function main() { + if (!process.env.HH_API_KEY) { + throw new Error("HH_API_KEY environment variable is not set."); + } + if (!process.env.HH_PROJECT) { + throw new Error("HH_PROJECT environment variable is not set."); + } + + const tracer = await HoneyHiveTracer.init({ + apiKey: process.env.HH_API_KEY, + project: process.env.HH_PROJECT, + serverUrl: process.env.HH_API_URL + }); + + const currentSessionId = tracer.sessionId; + + tracer.enrichSession({ + metrics: { + "json_validated": true, + "num_actions": 10, + // any other custom fields and values as you need + "step_evals": [ + { + "invalid_grammar": false, + "unable_to_locate_UI": true + } + ], + } + }); + + // Wait 5 seconds for data propagation + await new Promise(resolve => setTimeout(resolve, 5000)); + + const sdk = new HoneyHive({ + bearerAuth: process.env.HH_API_KEY, + serverURL: process.env.HH_API_URL + }); + + console.log(`Fetching events for session ID: ${currentSessionId}`); + try { + const res = await sdk.events.getEvents({ + project: process.env.HH_PROJECT, + filters: [ + { + field: "event_type", + value: "session", + operator: Operator.Is + }, + { + field: "session_id", + value: currentSessionId, + operator: Operator.Is + } + ], + }); + + // Assert that events exist and the first event has the expected metrics + assert(res.events && res.events.length > 0, `No events found for session ${currentSessionId}`); + const firstEvent = res.events[0]; + assert(firstEvent.metrics, `Metrics not found for the first event in session ${currentSessionId}`); + assert.strictEqual(firstEvent.metrics.json_validated, true, "json_validated metric does not match"); + assert.strictEqual(firstEvent.metrics.num_actions, 10, "num_actions metric does not match"); + assert(Array.isArray(firstEvent.metrics.step_evals), "step_evals metric is not an array"); + assert.strictEqual(firstEvent.metrics.step_evals[0]?.invalid_grammar, false, "step_evals[0].invalid_grammar does not match"); + assert.strictEqual(firstEvent.metrics.step_evals[0]?.unable_to_locate_UI, true, "step_evals[0].unable_to_locate_UI does not match"); + + console.log(`Events for session ${currentSessionId}:`, res.events); + + } catch (error) { + console.error(`Error fetching events for session ${currentSessionId}:`, error); + throw error; // Re-throw the error to fail the test + } + + console.log('Finished fetching events.'); +} + +export { main }; \ No newline at end of file diff --git a/tests/integration/experiments_quickstart.ts b/tests/integration/experiments_quickstart.ts new file mode 100644 index 0000000..723a9df --- /dev/null +++ b/tests/integration/experiments_quickstart.ts @@ -0,0 +1,147 @@ +import { evaluate } from "honeyhive"; +import { OpenAI } from 'openai'; +import { HoneyHive } from "honeyhive"; +import assert from "assert"; +import { Operator } from "honeyhive/models/components/index.js"; + +const openai = new OpenAI({ apiKey: process.env.OPENAI_KEY }); + +// Create function to be evaluated +// input -> parameter to which datapoint or json value will be passed +export async function functionToEvaluate(input: Record, ground_truths: Record) { + + try { + const response = await openai.chat.completions.create({ + model: "gpt-4", + messages: [ + { + role: 'system', + content: `You are an expert analyst specializing in ${input.product_type} market trends.` + }, + { + role: 'user', + content: `Could you provide an analysis of the current market performance and consumer reception of ${input.product_type} in ${input.region}? Please include any notable trends or challenges specific to this region.` + } + ], + }); + + // Output -> session output + return response.choices[0].message; + } catch (error) { + console.error('Error making GPT-4 call:', error); + throw error; + } +} + +const dataset = [ + { + "inputs": { + "product_type": "electric vehicles", + "region": "western europe" + }, + "ground_truths": { + "response": "As of 2023, the electric vehicle (EV) ... ", + } + }, + { + "inputs": { + "product_type": "gaming consoles", + "region": "north america" + }, + "ground_truths": { + "response": "As of 2023, the gaming console market ... ", + } + }, + { + "inputs": { + "product_type": "smart home devices", + "region": "australia and new zealand" + }, + "ground_truths": { + "response": "As of 2023, the market for smart home devices in Australia and New Zealand ... ", + } + } +] + +// Sample evaluator that returns fixed metrics +function sampleEvaluator(output: any, input: Record, ground_truths: Record) { + // Code here + return { + sample_metric: 0.5, + sample_metric_2: true + }; +} + +async function main() { + + if (!process.env.HH_API_KEY) { + throw new Error("HH_API_KEY environment variable is not set."); + } + if (!process.env.HH_PROJECT) { + throw new Error("HH_PROJECT environment variable is not set."); + } + if (!process.env.OPENAI_API_KEY) { + throw new Error("OPENAI_API_KEY environment variable is not set."); + } + + const result = await evaluate({ + function: functionToEvaluate, // Function to be evaluated + apiKey: process.env.HH_API_KEY, + project: process.env.HH_PROJECT, + name: 'Sample Experiment', + dataset: dataset, // to be passed for json_list + evaluators: [sampleEvaluator], // to compute client-side metrics on each run + serverUrl: process.env.HH_API_URL + }); + + assert(result, "Evaluate result should not be undefined"); + const sessionIds = result.sessionIds; + assert(sessionIds && sessionIds.length > 0, "sessionIds should be populated"); + // Wait 5 seconds for data propagation + await new Promise(resolve => setTimeout(resolve, 10000)); + const sdk = new HoneyHive({ + bearerAuth: process.env.HH_API_KEY, + }); + + + for (const sessionId of sessionIds) { + console.log(`Fetching events for session ID: ${sessionId}`); + try { + const res = await sdk.events.getEvents({ + project: process.env.HH_PROJECT, + filters: [ + { + field: "event_type", + value: "session", + operator: Operator.Is + }, + { + field: "session_id", + value: sessionId, + operator: Operator.Is + } + ], + }); + + // Assert that events exist and the first event has the expected metrics + assert(res.events && res.events.length > 0, `No events found for session ${sessionId}`); + const firstEvent = res.events[0]; + assert(firstEvent.metrics, `Metrics not found for the first event in session ${sessionId}`); + assert(firstEvent.metrics.sampleEvaluator, `sampleEvaluator metric not found in the first event for session ${sessionId}`); + assert.deepStrictEqual(firstEvent.metrics.sampleEvaluator, + { sample_metric: 0.5, sample_metric_2: true }, + `Metrics for sampleEvaluator in session ${sessionId} do not match the expected values.` + ); + + console.log(`Events for session ${sessionId}:`, res.events); + + } catch (error) { + console.error(`Error fetching events for session ${sessionId}:`, error); + } + } + + console.log('Finished fetching events for all sessions.'); + +} + +export { main }; diff --git a/tests/integration/introduction_quickstart.ts b/tests/integration/introduction_quickstart.ts new file mode 100644 index 0000000..e410308 --- /dev/null +++ b/tests/integration/introduction_quickstart.ts @@ -0,0 +1,74 @@ +import { HoneyHiveTracer, HoneyHive } from "honeyhive"; +import assert from "assert"; +import { Operator } from "honeyhive/models/components/index.js"; + + +async function main() { + // Environment variable checks + if (!process.env.HH_API_KEY) { + throw new Error("HH_API_KEY environment variable is not set."); + } + if (!process.env.HH_PROJECT) { + throw new Error("HH_PROJECT environment variable is not set."); + } + // Optional variables - provide defaults or handle absence if needed + const source = process.env.HH_SOURCE; // Optional + const sessionName = process.env.HH_SESSION; // Optional + const serverURL = process.env.HH_API_URL; // Optional + + // Initialize tracer + const tracer = await HoneyHiveTracer.init({ + apiKey: process.env.HH_API_KEY, + project: process.env.HH_PROJECT, + source: source, + sessionName: sessionName, + serverUrl: serverURL // Correct property name is serverUrl + }); + + const currentSessionId = tracer.sessionId; + console.log(`Initialized tracer with session ID: ${currentSessionId}`); + + // Wait for data propagation + await new Promise(resolve => setTimeout(resolve, 10000)); + + // Initialize SDK + const sdk = new HoneyHive({ + bearerAuth: process.env.HH_API_KEY, + serverURL: serverURL // Correct property name is serverURL here + }); + + // Fetch events for the session + console.log(`Fetching events for session ID: ${currentSessionId}`); + try { + const res = await sdk.events.getEvents({ + project: process.env.HH_PROJECT, + filters: [ + { + field: "event_type", + value: "session", + operator: Operator.Is + }, + { + field: "session_id", // Filter uses session_id + value: currentSessionId, + operator: Operator.Is + } + ], + }); + + // Assertions + assert(res.events, `Events response is undefined for session ${currentSessionId}`); + assert(res.events.length > 0, `No events found for session ${currentSessionId}`); + const firstEvent = res.events[0]; + assert.strictEqual(firstEvent.sessionId, currentSessionId, `Event session ID ${firstEvent.sessionId} does not match expected ${currentSessionId}`); // Event object uses sessionId + console.log(`Successfully verified session start event for session ID: ${currentSessionId}`); + + } catch (error) { + console.error(`Error fetching or verifying events for session ${currentSessionId}:`, error); + throw error; // Re-throw the error to fail the test + } + + console.log('Test finished successfully.'); +} + +export { main }; diff --git a/tests/integration/multi_step_trace.ts b/tests/integration/multi_step_trace.ts new file mode 100644 index 0000000..9ed12dd --- /dev/null +++ b/tests/integration/multi_step_trace.ts @@ -0,0 +1,197 @@ +import { HoneyHiveTracer, traceTool, traceModel, traceChain, enrichSpan, enrichSession, HoneyHive } from "honeyhive"; +import assert from "assert"; +import { Operator, Event } from "honeyhive/models/components/index.js"; // Import Event type + +// Keep interfaces used in the functions (copied from new_multi_step_trace.ts) +interface MedicalDocument { + docs: string[]; + response: string; +} + +interface RagPipelineMetrics { + num_retrieved_docs: number; + query_length: number; +} + +// Initialize tracer function (adapted from new_multi_step_trace.ts) +async function initializeTracer() { + // Environment variable checks (can be kept or rely on SDK defaults/env vars) + if (!process.env.HH_API_KEY) { + throw new Error("HH_API_KEY environment variable is not set."); + } + if (!process.env.HH_PROJECT) { + throw new Error("HH_PROJECT environment variable is not set."); + } + const serverURL = process.env.HH_API_URL; // Optional + + return await HoneyHiveTracer.init({ + apiKey: process.env.HH_API_KEY, + project: process.env.HH_PROJECT, + sessionName: "docs-multi-step-trace-test", // Use a distinct session name for this test + serverUrl: serverURL, + verbose: true, // Enable verbose logging for debugging if needed + }); +} + +// Main function to initialize tracer, run the traced logic, and verify +async function main() { + const tracer = await initializeTracer(); + const currentSessionId = tracer.sessionId; // Capture session ID + console.log(`Initialized tracer with session ID: ${currentSessionId}`); + let executionSuccess = false; + + // Define the functions based on the provided docs snippet + // Define the get_relevant_docs function with traceTool + const getRelevantDocs = traceTool(function getRelevantDocs( + query: string + ): string[] { + console.log("Executing getRelevantDocs (from docs snippet)"); + const medicalDocs = [ + "Regular exercise reduces diabetes risk by 30%. Daily walking is recommended.", + "Studies show morning exercises have better impact on blood sugar levels." + ]; + + enrichSpan({ + metrics: { retrieval_relevance: 0.5 } + }); + + return medicalDocs; + }); + + // Define generateResponse with traceModel + const generateResponse = traceModel(function generateResponse( + docs: string[], + query: string + ): string { + console.log("Executing generateResponse (from docs snippet)"); + const prompt = `Question: ${query}\nContext: ${docs}\nAnswer:`; + const response = "This is a test response."; + + enrichSpan({ + metrics: { contains_citations: true } + }); + + return response; + }); + + // Define ragPipeline with traceChain + const ragPipeline = traceChain(function ragPipeline( + query: string + ): MedicalDocument { + console.log("Executing ragPipeline (from docs snippet)"); + const docs = getRelevantDocs(query); + const response = generateResponse(docs, query); // Synchronous call in this example + + enrichSession({ + metrics: { + rag_pipeline: { + num_retrieved_docs: docs.length, + query_length: query.split(" ").length + } as RagPipelineMetrics // Use type assertion for clarity if needed + } + }); + + return { docs, response }; + }); + + + try { + // --- Main Execution Logic (Adapted from docs snippet) --- + // Wrap the execution in tracer.trace() to establish context + console.log("Executing pipeline within tracer.trace()..."); + await tracer.trace(async () => { + const query = "How does exercise affect diabetes?"; + // Note: ragPipeline is synchronous in this example, but await is harmless + // If ragPipeline truly becomes async, the await is necessary. + const result = ragPipeline(query); + console.log("Pipeline executed inside tracer.trace(), result:", result); + }); + executionSuccess = true; // Mark execution as successful if no error + console.log("Traced execution finished."); + + // Ensure data is sent before verification (copied from new_multi_step_trace.ts) + console.log("Waiting for data propagation after flush..."); + await tracer.flush(); // Make sure flush is called + console.log("Tracer flushed."); + await new Promise(resolve => setTimeout(resolve, 15000)); // Wait like original script + + // --- Verification Logic (Copied from new_multi_step_trace.ts) --- + console.log(`Starting verification for session ID: ${currentSessionId}`); + const sdk = new HoneyHive({ + bearerAuth: process.env.HH_API_KEY!, // Assume non-null from earlier check + serverURL: process.env.HH_API_URL // Optional + }); + + console.log(`Fetching events for session ID: ${currentSessionId}`); + const res = await sdk.events.getEvents({ + project: process.env.HH_PROJECT!, // Assume non-null + filters: [ + { + field: "session_id", + value: currentSessionId, + operator: Operator.Is + } + ], + }); + + // Assertions + assert(res.events, `Events response is undefined for session ${currentSessionId}`); + // Expecting >= 5 events: Session Start, Outer trace span, rag_pipeline, get_relevant_docs, generate_response + // The outer tracer.trace() adds an event compared to new_multi_step_trace.ts + assert(res.events.length >= 5, `Expected at least 5 events for session ${currentSessionId}, found ${res.events.length}`); + console.log(`Found ${res.events.length} events.`); + + // Check for span-level metrics + let spanMetricsFound = { + retrieval_relevance: false, + contains_citations: false + }; + // Find specific events for metrics, excluding the outer trace span and session start + const relevantEvents = res.events.filter(e => e.eventName === 'getRelevantDocs' || e.eventName === 'generateResponse'); + + relevantEvents.forEach((event: Event) => { + if (event.metrics) { + if (event.eventName === 'getRelevantDocs' && 'retrieval_relevance' in event.metrics) { + spanMetricsFound.retrieval_relevance = true; + console.log(`Found retrieval_relevance metric in event: ${event.eventId} (${event.eventName})`); + assert.strictEqual(event.metrics.retrieval_relevance, 0.5, "retrieval_relevance value mismatch"); + } + if (event.eventName === 'generateResponse' && 'contains_citations' in event.metrics) { + spanMetricsFound.contains_citations = true; + console.log(`Found contains_citations metric in event: ${event.eventId} (${event.eventName})`); + assert.strictEqual(event.metrics.contains_citations, true, "contains_citations value mismatch"); + } + } + }); + + + assert(spanMetricsFound.retrieval_relevance, "'retrieval_relevance' metric not found in 'getRelevantDocs' event"); + assert(spanMetricsFound.contains_citations, "'contains_citations' metric not found in 'generateResponse' event"); + console.log("Span-level metrics verified."); + + // Check for session-level metrics (should be in the session event) + const sessionEvent = res.events.find((e: Event) => e.eventType === 'session'); + assert(sessionEvent, "Session start event not found"); + assert(sessionEvent.metrics, "Metrics not found in session start event"); + console.log("Session event metrics:", sessionEvent.metrics); + assert(sessionEvent.metrics.rag_pipeline, "'rag_pipeline' metric not found in session start event metrics"); + const ragPipelineMetrics = sessionEvent.metrics.rag_pipeline as RagPipelineMetrics; // Use type assertion + assert(ragPipelineMetrics, "'rag_pipeline' metric structure is invalid"); + assert.strictEqual(ragPipelineMetrics.num_retrieved_docs, 2, "num_retrieved_docs does not match"); + assert.strictEqual(ragPipelineMetrics.query_length, 5, "query_length does not match"); + console.log("Session-level metrics verified."); + console.log('Verification finished successfully.'); + // --- End Verification Logic --- + + return true; // Return true only if both execution and verification succeed + + } catch (error) { + console.error(`Error during traced execution or verification for session ${currentSessionId}:`, error); + // If execution failed, the error is already caught. + // If verification failed, we log the error here. + return false; // Indicate failure + } + // No finally block needed here as flush is now inside the try block before verification +} + +export { main }; \ No newline at end of file diff --git a/tests/integration/multi_step_trace_legacy.ts b/tests/integration/multi_step_trace_legacy.ts new file mode 100644 index 0000000..45b4fbe --- /dev/null +++ b/tests/integration/multi_step_trace_legacy.ts @@ -0,0 +1,165 @@ +import { HoneyHiveTracer, HoneyHive } from 'honeyhive'; +import assert from "assert"; +import { Operator, Event } from "honeyhive/models/components/index.js"; // Import Event type + +// Keep interfaces used in the functions +interface MedicalDocument { + docs: string[]; + response: string; +} + +interface RagPipelineMetrics { + num_retrieved_docs: number; + query_length: number; +} + +async function main() { + // Environment variable checks + if (!process.env.HH_API_KEY) { + throw new Error("HH_API_KEY environment variable is not set."); + } + if (!process.env.HH_PROJECT) { + throw new Error("HH_PROJECT environment variable is not set."); + } + const serverURL = process.env.HH_API_URL; // Optional + + const tracer = await HoneyHiveTracer.init({ + apiKey: process.env.HH_API_KEY, + project: process.env.HH_PROJECT, + sessionName: "multi-step-trace-test", // Use a distinct session name for the test + serverUrl: serverURL + }); + + const currentSessionId = tracer.sessionId; + console.log(`Initialized tracer with session ID: ${currentSessionId}`); + + // Define the get_relevant_docs function with tracing + const getRelevantDocs = tracer.traceFunction()( + function getRelevantDocs(query: string): string[] { + console.log("Executing getRelevantDocs"); + const medicalDocs = [ + "Regular exercise reduces diabetes risk by 30%. Daily walking is recommended.", + "Studies show morning exercises have better impact on blood sugar levels." + ]; + + tracer.enrichSpan({ + metrics: { retrieval_relevance: 0.5 } + }); + + return medicalDocs; + } + ); + + const generateResponse = tracer.traceFunction()( + function generateResponse(docs: string[], query: string): string { + console.log("Executing generateResponse"); + const prompt = `Question: ${query}\nContext: ${docs}\nAnswer:`; + const response = "This is a test response."; + + tracer.enrichSpan({ + metrics: { contains_citations: true } + }); + + return response; + } + ); + + const ragPipeline = tracer.traceFunction()( + function ragPipeline(query: string): MedicalDocument { + console.log("Executing ragPipeline"); + const docs = getRelevantDocs(query); + const response = generateResponse(docs, query); + + tracer.enrichSession({ + metrics: { + rag_pipeline: { + num_retrieved_docs: docs.length, + query_length: query.split(" ").length + } // No need for 'as RagPipelineMetrics' if types match + } + }); + + return { docs, response }; + } + ); + + // Execute the pipeline + const query = "How does exercise affect diabetes?"; + console.log("Executing pipeline..."); + const result = ragPipeline(query); + console.log("Pipeline executed, result:", result); + + // Wait for data propagation + console.log("Waiting for data propagation..."); + await new Promise(resolve => setTimeout(resolve, 15000)); + + // Initialize SDK + const sdk = new HoneyHive({ + bearerAuth: process.env.HH_API_KEY, + serverURL: serverURL + }); + + // Fetch events for the session + console.log(`Fetching events for session ID: ${currentSessionId}`); + try { + const res = await sdk.events.getEvents({ + project: process.env.HH_PROJECT, + filters: [ + { + field: "session_id", + value: currentSessionId, + operator: Operator.Is + } + ], + }); + + // Assertions + assert(res.events, `Events response is undefined for session ${currentSessionId}`); + // Expecting >= 4 events: Session Start, rag_pipeline, get_relevant_docs, generate_response + assert(res.events.length >= 4, `Expected at least 4 events for session ${currentSessionId}, found ${res.events.length}`); + console.log(`Found ${res.events.length} events.`); + + // Check for span-level metrics + let spanMetricsFound = { + retrieval_relevance: false, + contains_citations: false + }; + res.events.forEach((event: Event) => { + if (event.metrics) { + if ('retrieval_relevance' in event.metrics) { + spanMetricsFound.retrieval_relevance = true; + console.log(`Found retrieval_relevance metric in event: ${event.eventId}`); + } + if ('contains_citations' in event.metrics) { + spanMetricsFound.contains_citations = true; + console.log(`Found contains_citations metric in event: ${event.eventId}`); + } + } + }); + + assert(spanMetricsFound.retrieval_relevance, "'retrieval_relevance' metric not found in any event"); + assert(spanMetricsFound.contains_citations, "'contains_citations' metric not found in any event"); + console.log("Span-level metrics verified."); + + // Check for session-level metrics (should be in the session event) + const sessionEvent = res.events.find((e: Event) => e.eventType === 'session'); + assert(sessionEvent, "Session start event not found"); + assert(sessionEvent.metrics, "Metrics not found in session start event"); + console.log("Session event metrics:", sessionEvent.metrics); + assert(sessionEvent.metrics.rag_pipeline, "'rag_pipeline' metric not found in session start event metrics"); + // Use type assertion or check existence before accessing nested properties + const ragPipelineMetrics = sessionEvent.metrics.rag_pipeline as RagPipelineMetrics; + assert(ragPipelineMetrics, "'rag_pipeline' metric structure is invalid"); + assert.strictEqual(ragPipelineMetrics.num_retrieved_docs, 2, "num_retrieved_docs does not match"); + assert.strictEqual(ragPipelineMetrics.query_length, 5, "query_length does not match"); + console.log("Session-level metrics verified."); + + } catch (error) { + console.error(`Error fetching or verifying events for session ${currentSessionId}:`, error); + throw error; // Re-throw the error to fail the test + } + + console.log('Test finished successfully.'); +} + +export { main }; \ No newline at end of file diff --git a/tests/integration/setting_config.ts b/tests/integration/setting_config.ts new file mode 100644 index 0000000..9d83735 --- /dev/null +++ b/tests/integration/setting_config.ts @@ -0,0 +1,152 @@ +import { HoneyHiveTracer, traceTool, enrichSpan, HoneyHive } from "honeyhive"; +import assert from "assert"; +import { Operator, Event } from "honeyhive/models/components/index.js"; + +// Define types for the prompt template structure (from docs snippet) +interface PromptMessage { + role: string; + content: string; +} + +interface PromptTemplate { + template: PromptMessage[]; + prompt: PromptMessage[]; +} + +// Initialize tracer function (adapted from new_setting_config.ts) +async function initializeTracer() { + // Environment variable checks + if (!process.env.HH_API_KEY) { + throw new Error("HH_API_KEY environment variable is not set."); + } + if (!process.env.HH_PROJECT) { + throw new Error("HH_PROJECT environment variable is not set."); + } + const serverURL = process.env.HH_API_URL; // Optional + + return await HoneyHiveTracer.init({ + apiKey: process.env.HH_API_KEY, + project: process.env.HH_PROJECT, + sessionName: "docs-setting-config-test", // Distinct session name + serverUrl: serverURL, + verbose: true, + }); +} + +// Main function to initialize tracer, run the traced logic, and verify +async function main() { + const tracer = await initializeTracer(); + const currentSessionId = tracer.sessionId; // Capture session ID + console.log(`Initialized tracer with session ID: ${currentSessionId}`); + + // Define the function to be traced (from docs snippet) + const myTracedFunction = traceTool( + function my_function( // Function name is used as span name + input: string, + prompt_template: PromptTemplate + ) { + console.log("Executing my_function (from docs snippet)"); + // Add configuration specific to this span + enrichSpan({ + config: { + template: prompt_template.template, + prompt: prompt_template.prompt, + hyperparams: { + temperature: 0.5, + max_tokens: 100, + top_p: 0.9, + top_k: 50, + } + } + }); + + // Your function code here + const response = `Processed input: ${input}`; + console.log("my_function finished"); + return response; + } + ); + + // Define the prompt data (from docs snippet execution block) + const prompt_template_data: PromptTemplate = { + template: [ + { role: "system", content: "You are a helpful AI assistant." }, + { role: "user", content: "Write a short poem about programming." } + ], + prompt: [ + { role: "system", content: "You are a helpful AI assistant." }, + { role: "user", content: "Write a short poem about programming." } + ] + }; + + try { + // --- Main Execution Logic (from docs snippet) --- + // Wrap the execution in tracer.trace() to establish context + console.log("Calling myTracedFunction within tracer.trace()..."); + await tracer.trace(async () => { + // Execute the traced function within the trace context + const result = myTracedFunction("Some input data", prompt_template_data); + console.log("myTracedFunction call returned:", result); + }); + console.log("Traced execution finished."); + + // Ensure data is sent before verification (copied from new_setting_config.ts) + console.log("Waiting for data propagation after flush..."); + await tracer.flush(); + console.log("Tracer flushed."); + await new Promise(resolve => setTimeout(resolve, 5000)); // Wait + + // --- Verification Logic (Adapted from new_setting_config.ts) --- + console.log(`Starting verification for session ID: ${currentSessionId}`); + const sdk = new HoneyHive({ + bearerAuth: process.env.HH_API_KEY!, + serverURL: process.env.HH_API_URL + }); + + console.log(`Fetching events for session ID: ${currentSessionId}`); + const res = await sdk.events.getEvents({ + project: process.env.HH_PROJECT!, + filters: [ + { + field: "session_id", + value: currentSessionId, + operator: Operator.Is + } + ], + }); + + // Assertions + assert(res.events, `Events response is undefined for session ${currentSessionId}`); + // Expecting at least 3 events: Session Start, outer tracer.trace span, my_function span + assert(res.events.length >= 3, `Expected at least 3 events for session ${currentSessionId}, found ${res.events.length}`); + console.log(`Found ${res.events.length} events.`); + + // Find the specific event for the traced function + const myFunctionEvent = res.events.find((e: Event) => e.eventName === 'my_function'); // Use function name + assert(myFunctionEvent, "'my_function' event not found"); + console.log(`Found 'my_function' event: ${myFunctionEvent.eventId}`); + assert(myFunctionEvent.config, "Config not found in 'my_function' event"); + console.log("'my_function' event config:", myFunctionEvent.config); + + // Verify the config content against the data used + assert.deepStrictEqual(myFunctionEvent.config.template, prompt_template_data.template, "Config template does not match"); + assert.deepStrictEqual(myFunctionEvent.config.prompt, prompt_template_data.prompt, "Config prompt does not match"); + assert.deepStrictEqual(myFunctionEvent.config.hyperparams, { + temperature: 0.5, + max_tokens: 100, + top_p: 0.9, + top_k: 50, + }, "Config hyperparams do not match"); + console.log("Config content verified successfully."); + // --- End Verification Logic --- + + console.log('Test finished successfully.'); + return true; // Return true only if both execution and verification succeed + + } catch (error) { + console.error(`Error during traced execution or verification for session ${currentSessionId}:`, error); + return false; // Indicate failure + } +} + +export { main }; \ No newline at end of file diff --git a/tests/integration/setting_config_legacy.ts b/tests/integration/setting_config_legacy.ts new file mode 100644 index 0000000..c1ade8e --- /dev/null +++ b/tests/integration/setting_config_legacy.ts @@ -0,0 +1,138 @@ +import { HoneyHiveTracer, HoneyHive } from "honeyhive"; +import assert from "assert"; +import { Operator, Event } from "honeyhive/models/components/index.js"; + +// Define types for the prompt template structure +interface PromptMessage { + role: string; + content: string; +} + +interface PromptTemplate { + template: PromptMessage[]; + prompt: PromptMessage[]; +} + +async function main() { + // Environment variable checks + if (!process.env.HH_API_KEY) { + throw new Error("HH_API_KEY environment variable is not set."); + } + if (!process.env.HH_PROJECT) { + throw new Error("HH_PROJECT environment variable is not set."); + } + const serverURL = process.env.HH_API_URL; // Optional + + const tracer = await HoneyHiveTracer.init({ + apiKey: process.env.HH_API_KEY, + project: process.env.HH_PROJECT, + sessionName: "setting-config-test", // Distinct session name + serverUrl: serverURL + }); + + const currentSessionId = tracer.sessionId; + console.log(`Initialized tracer with session ID: ${currentSessionId}`); + + // Define the prompt template object + const prompt_template: PromptTemplate = { + template: [ + {role: "system", content: "You are a helpful AI assistant."}, + {role: "user", content: "Write a short poem about programming."} + ], + prompt: [ + {role: "system", content: "You are a helpful AI assistant."}, + {role: "user", content: "Write a short poem about programming."} + ] + }; + + const myTracedFunction = tracer.traceFunction({eventName: "my_function"})( + function ( + input: string, + pt: PromptTemplate // Use the defined type + ) { + console.log("Executing myTracedFunction"); + // span enrichment + tracer.enrichSpan({ + config: { + template: pt.template, + prompt: pt.prompt, + hyperparams: { + temperature: 0.5, + max_tokens: 100, + top_p: 0.9, + top_k: 50, + } + } + }); + + // Your function code here (mock response) + const response = "This is a mock response."; + console.log("myTracedFunction finished"); + return response; + } + ); + + // Execute the traced function + console.log("Calling myTracedFunction..."); + const result = myTracedFunction("This is a mock input", prompt_template); + console.log("myTracedFunction call returned:", result); + + // Wait for data propagation + console.log("Waiting for data propagation..."); + await new Promise(resolve => setTimeout(resolve, 15000)); + + // Initialize SDK + const sdk = new HoneyHive({ + bearerAuth: process.env.HH_API_KEY, + serverURL: serverURL + }); + + // Fetch events for the session + console.log(`Fetching events for session ID: ${currentSessionId}`); + try { + const res = await sdk.events.getEvents({ + project: process.env.HH_PROJECT, + filters: [ + { + field: "session_id", + value: currentSessionId, + operator: Operator.Is + } + ], + }); + + // Assertions + assert(res.events, `Events response is undefined for session ${currentSessionId}`); + // Expecting at least 2 events: Session Start, my_function trace + assert(res.events.length >= 2, `Expected at least 2 events for session ${currentSessionId}, found ${res.events.length}`); + console.log(`Found ${res.events.length} events.`); + + // Find the event for the 'my_function' trace + // Note: The event name defaults to the function name if not specified in traceFunction + // Since we passed "my_function" to traceFunction, we look for that. + const myFunctionEvent = res.events.find((e: Event) => e.eventName === 'my_function'); + assert(myFunctionEvent, "'my_function' event not found"); + console.log(`Found 'my_function' event: ${myFunctionEvent.eventId}`); + assert(myFunctionEvent.config, "Config not found in 'my_function' event"); + console.log("'my_function' event config:", myFunctionEvent.config); + + // Verify the config contents using deepStrictEqual for objects/arrays + assert.deepStrictEqual(myFunctionEvent.config.template, prompt_template.template, "Config template does not match"); + assert.deepStrictEqual(myFunctionEvent.config.prompt, prompt_template.prompt, "Config prompt does not match"); + assert.deepStrictEqual(myFunctionEvent.config.hyperparams, { + temperature: 0.5, + max_tokens: 100, + top_p: 0.9, + top_k: 50, + }, "Config hyperparams do not match"); + console.log("Config content verified successfully."); + + } catch (error) { + console.error(`Error fetching or verifying events for session ${currentSessionId}:`, error); + throw error; // Re-throw the error to fail the test + } + + console.log('Test finished successfully.'); +} + +export { main }; \ No newline at end of file diff --git a/tests/integration/setting_metadata.ts b/tests/integration/setting_metadata.ts new file mode 100644 index 0000000..502db15 --- /dev/null +++ b/tests/integration/setting_metadata.ts @@ -0,0 +1,122 @@ +import { HoneyHiveTracer, traceTool, enrichSpan, HoneyHive } from "honeyhive"; +import assert from "assert"; +import { Operator, Event } from "honeyhive/models/components/index.js"; + +// Initialize tracer function (adapted from new_setting_metadata.ts) +async function initializeTracer() { + // Environment variable checks + if (!process.env.HH_API_KEY) { + throw new Error("HH_API_KEY environment variable is not set."); + } + if (!process.env.HH_PROJECT) { + throw new Error("HH_PROJECT environment variable is not set."); + } + const serverURL = process.env.HH_API_URL; // Optional + + return await HoneyHiveTracer.init({ + apiKey: process.env.HH_API_KEY, + project: process.env.HH_PROJECT, + sessionName: "docs-setting-metadata-test", // Distinct session name + serverUrl: serverURL, + verbose: true, + }); +} + +// Main function to initialize tracer, run the traced logic, and verify +async function main() { + const tracer = await initializeTracer(); + const currentSessionId = tracer.sessionId; // Capture session ID + console.log(`Initialized tracer with session ID: ${currentSessionId}`); + + // Define the traced function using traceTool (from docs snippet) + const myTracedFunction = traceTool( + function my_function ( // Function name is used as span name + input: string, + something: any + ) { + console.log("Executing my_function (from docs snippet)"); + // Add metadata specific to this span + enrichSpan({ + metadata: { + "experiment-id": 12345, + "something": something, + // any other custom fields and values as you need + } + }); + + // Your function code here (mock response) + const response = `Processed input: ${input}`; + console.log("my_function finished"); + return response; + } + ); + + // Define the metadata value to be used (from docs snippet execution block) + const metadataValue = "some-metadata"; + + try { + // --- Main Execution Logic (from docs snippet) --- + // Wrap the execution in tracer.trace() to establish context + console.log("Calling myTracedFunction within tracer.trace()..."); + await tracer.trace(async () => { + // Execute the traced function within the trace context + const result = myTracedFunction("This is a mock input", metadataValue); + console.log("myTracedFunction call returned:", result); + }); + console.log("Traced execution finished."); + + + // Ensure data is sent before verification (copied from new_setting_metadata.ts) + console.log("Waiting for data propagation after flush..."); + await tracer.flush(); + console.log("Tracer flushed."); + await new Promise(resolve => setTimeout(resolve, 15000)); // Wait + + // --- Verification Logic (Adapted from new_setting_metadata.ts) --- + console.log(`Starting verification for session ID: ${currentSessionId}`); + const sdk = new HoneyHive({ + bearerAuth: process.env.HH_API_KEY!, + serverURL: process.env.HH_API_URL + }); + + console.log(`Fetching events for session ID: ${currentSessionId}`); + const res = await sdk.events.getEvents({ + project: process.env.HH_PROJECT!, + filters: [ + { + field: "session_id", + value: currentSessionId, + operator: Operator.Is + } + ], + }); + + // Assertions + assert(res.events, `Events response is undefined for session ${currentSessionId}`); + // Expecting at least 3 events: Session Start, outer tracer.trace span, my_function span + assert(res.events.length >= 3, `Expected at least 3 events for session ${currentSessionId}, found ${res.events.length}`); + console.log(`Found ${res.events.length} events.`); + + const myFunctionEvent = res.events.find((e: Event) => e.eventName === 'my_function'); // Use function name + assert(myFunctionEvent, "'my_function' event not found"); + console.log(`Found 'my_function' event: ${myFunctionEvent.eventId}`); + assert(myFunctionEvent.metadata, "Metadata not found in 'my_function' event"); + console.log("'my_function' event metadata:", myFunctionEvent.metadata); + + // Verify the metadata contents + assert.strictEqual(myFunctionEvent.metadata["experiment-id"], 12345, "Metadata 'experiment-id' does not match"); + // Use the value defined before the try block for verification + assert.strictEqual(myFunctionEvent.metadata["something"], metadataValue, "Metadata 'something' does not match"); + console.log("Metadata content verified successfully."); + // --- End Verification Logic --- + + console.log('Test finished successfully.'); + return true; // Return true only if both execution and verification succeed + + } catch (error) { + console.error(`Error during traced execution or verification for session ${currentSessionId}:`, error); + return false; // Indicate failure + } +} + +export { main }; \ No newline at end of file diff --git a/tests/integration/setting_metadata_legacy.ts b/tests/integration/setting_metadata_legacy.ts new file mode 100644 index 0000000..b87088c --- /dev/null +++ b/tests/integration/setting_metadata_legacy.ts @@ -0,0 +1,104 @@ +import { HoneyHiveTracer, HoneyHive } from "honeyhive"; +import assert from "assert"; +import { Operator, Event } from "honeyhive/models/components/index.js"; + +async function main() { + // Environment variable checks + if (!process.env.HH_API_KEY) { + throw new Error("HH_API_KEY environment variable is not set."); + } + if (!process.env.HH_PROJECT) { + throw new Error("HH_PROJECT environment variable is not set."); + } + const serverURL = process.env.HH_API_URL; // Optional + + const tracer = await HoneyHiveTracer.init({ + apiKey: process.env.HH_API_KEY, + project: process.env.HH_PROJECT, + sessionName: "setting-metadata-test", // Distinct session name + serverUrl: serverURL + }); + + const currentSessionId = tracer.sessionId; + console.log(`Initialized tracer with session ID: ${currentSessionId}`); + + const myTracedFunction = tracer.traceFunction({eventName: "my_function"})( + function ( + input: string, + something: any + ) { + console.log("Executing myTracedFunction"); + // span enrichment + tracer.enrichSpan({ + metadata: { + "experiment-id": 12345, + "something": something, + // any other custom fields and values as you need + } + }); + + // Your function code here (mock response) + const response = "This is a mock response."; + console.log("myTracedFunction finished"); + return response; + } + ); + + // Execute the traced function + const metadataValue = "some-metadata"; + console.log("Calling myTracedFunction..."); + const result = myTracedFunction("This is a mock input", metadataValue); + console.log("myTracedFunction call returned:", result); + + // Wait for data propagation + console.log("Waiting for data propagation..."); + await new Promise(resolve => setTimeout(resolve, 15000)); + + // Initialize SDK + const sdk = new HoneyHive({ + bearerAuth: process.env.HH_API_KEY, + serverURL: serverURL + }); + + // Fetch events for the session + console.log(`Fetching events for session ID: ${currentSessionId}`); + try { + const res = await sdk.events.getEvents({ + project: process.env.HH_PROJECT, + filters: [ + { + field: "session_id", + value: currentSessionId, + operator: Operator.Is + } + ], + }); + + // Assertions + assert(res.events, `Events response is undefined for session ${currentSessionId}`); + // Expecting at least 2 events: Session Start, my_function trace + assert(res.events.length >= 2, `Expected at least 2 events for session ${currentSessionId}, found ${res.events.length}`); + console.log(`Found ${res.events.length} events.`); + + // Find the event for the 'my_function' trace + const myFunctionEvent = res.events.find((e: Event) => e.eventName === 'my_function'); + assert(myFunctionEvent, "'my_function' event not found"); + console.log(`Found 'my_function' event: ${myFunctionEvent.eventId}`); + assert(myFunctionEvent.metadata, "Metadata not found in 'my_function' event"); + console.log("'my_function' event metadata:", myFunctionEvent.metadata); + + // Verify the metadata contents + // Use strictEqual for direct value comparison + assert.strictEqual(myFunctionEvent.metadata["experiment-id"], 12345, "Metadata 'experiment-id' does not match"); + assert.strictEqual(myFunctionEvent.metadata["something"], metadataValue, "Metadata 'something' does not match"); + console.log("Metadata content verified successfully."); + + } catch (error) { + console.error(`Error fetching or verifying events for session ${currentSessionId}:`, error); + throw error; // Re-throw the error to fail the test + } + + console.log('Test finished successfully.'); +} + +export { main }; \ No newline at end of file diff --git a/tests/integration/setting_user_feedback.ts b/tests/integration/setting_user_feedback.ts new file mode 100644 index 0000000..be086a6 --- /dev/null +++ b/tests/integration/setting_user_feedback.ts @@ -0,0 +1,124 @@ +import { HoneyHiveTracer, traceTool, enrichSpan, HoneyHive } from "honeyhive"; +import assert from "assert"; +import { Operator, Event } from "honeyhive/models/components/index.js"; + +// Initialize tracer function (adapted from previous examples) +async function initializeTracer() { + // Environment variable checks + if (!process.env.HH_API_KEY) { + throw new Error("HH_API_KEY environment variable is not set."); + } + if (!process.env.HH_PROJECT) { + throw new Error("HH_PROJECT environment variable is not set."); + } + const serverURL = process.env.HH_API_URL; // Optional + + return await HoneyHiveTracer.init({ + apiKey: process.env.HH_API_KEY, + project: process.env.HH_PROJECT, + sessionName: "docs-setting-user-feedback-test", // Distinct session name + serverUrl: serverURL, + verbose: true, + }); +} + +// Main function to initialize tracer, run the traced logic, and verify +async function main() { + const tracer = await initializeTracer(); + const currentSessionId = tracer.sessionId; // Capture session ID + console.log(`Initialized tracer with session ID: ${currentSessionId}`); + + // Define the traced function using traceTool (from docs snippet) + const myTracedFunction = traceTool(function my_llm_call( // Function name is used as span name + param1: string, + param2: string, + user_id: string | number + ) { + console.log("Executing my_llm_call (from docs snippet)"); + // Add feedback specific to this span + enrichSpan({ + feedback: { + "liked": true, + "comment": "The model hallucinated the capital of New York", + "user_id": user_id, + // optionally adding reference ground truth + "ground_truth": "The capital of New York is Albany", + // any other custom fields and values as you need + } + }); + + // Your function code here + const response = `Result with ${param1} and ${param2}`; + console.log("my_llm_call finished"); + return response; + }); + + // Define the user ID to be used (from docs snippet execution block) + const userId = "user-abc-123"; + + try { + // --- Main Execution Logic (from docs snippet) --- + // Wrap the execution in tracer.trace() to establish context + console.log("Calling myTracedFunction within tracer.trace()..."); + await tracer.trace(async () => { + // Execute the traced function within the trace context + const result = myTracedFunction("input1", "input2", userId); + console.log("myTracedFunction call returned:", result); + }); + console.log("Traced execution finished."); + + // Ensure data is sent before verification (copied from previous examples) + console.log("Waiting for data propagation after flush..."); + await tracer.flush(); + console.log("Tracer flushed."); + await new Promise(resolve => setTimeout(resolve, 15000)); // Wait + + // --- Verification Logic (Adapted from previous examples) --- + console.log(`Starting verification for session ID: ${currentSessionId}`); + const sdk = new HoneyHive({ + bearerAuth: process.env.HH_API_KEY!, + serverURL: process.env.HH_API_URL + }); + + console.log(`Fetching events for session ID: ${currentSessionId}`); + const res = await sdk.events.getEvents({ + project: process.env.HH_PROJECT!, + filters: [ + { + field: "session_id", + value: currentSessionId, + operator: Operator.Is + } + ], + }); + + // Assertions + assert(res.events, `Events response is undefined for session ${currentSessionId}`); + // Expecting at least 3 events: Session Start, outer tracer.trace span, my_llm_call span + assert(res.events.length >= 3, `Expected at least 3 events for session ${currentSessionId}, found ${res.events.length}`); + console.log(`Found ${res.events.length} events.`); + + const myLlmCallEvent = res.events.find((e: Event) => e.eventName === 'my_llm_call'); // Use function name + assert(myLlmCallEvent, "'my_llm_call' event not found"); + console.log(`Found 'my_llm_call' event: ${myLlmCallEvent.eventId}`); + assert(myLlmCallEvent.feedback, "Feedback not found in 'my_llm_call' event"); + console.log("'my_llm_call' event feedback:", myLlmCallEvent.feedback); + + // Verify the feedback contents + assert.strictEqual(myLlmCallEvent.feedback["liked"], true, "Feedback 'liked' does not match"); + assert.strictEqual(myLlmCallEvent.feedback["comment"], "The model hallucinated the capital of New York", "Feedback 'comment' does not match"); + assert.strictEqual(myLlmCallEvent.feedback["user_id"], userId, "Feedback 'user_id' does not match"); + assert.strictEqual(myLlmCallEvent.feedback["ground_truth"], "The capital of New York is Albany", "Feedback 'ground_truth' does not match"); + console.log("Feedback content verified successfully."); + // --- End Verification Logic --- + + console.log('Test finished successfully.'); + return true; // Return true only if both execution and verification succeed + + } catch (error) { + console.error(`Error during traced execution or verification for session ${currentSessionId}:`, error); + return false; // Indicate failure + } +} + +export { main }; \ No newline at end of file diff --git a/tests/integration/setting_user_properties.ts b/tests/integration/setting_user_properties.ts new file mode 100644 index 0000000..2373581 --- /dev/null +++ b/tests/integration/setting_user_properties.ts @@ -0,0 +1,122 @@ +import { HoneyHiveTracer, enrichSession, HoneyHive } from "honeyhive"; +import assert from "assert"; +import { Operator, Event } from "honeyhive/models/components/index.js"; + +// Define types for user properties (from docs snippet) +interface CustomUserProperties { + is_premium: boolean; + subscription_plan: string; + last_login: string; +} + +interface UserProperties { + user_id: string; + user_email: string; + user_properties: CustomUserProperties; +} + +// Initialize tracer function (adapted from previous examples) +async function initializeTracer() { + // Environment variable checks + if (!process.env.HH_API_KEY) { + throw new Error("HH_API_KEY environment variable is not set."); + } + if (!process.env.HH_PROJECT) { + throw new Error("HH_PROJECT environment variable is not set."); + } + const serverURL = process.env.HH_API_URL; // Optional + + return await HoneyHiveTracer.init({ + apiKey: process.env.HH_API_KEY, + project: process.env.HH_PROJECT, + sessionName: "docs-setting-user-props-test", // Distinct session name + serverUrl: serverURL, + verbose: true, + }); +} + +// Main function to initialize tracer, run the traced logic, and verify +async function main() { + const tracer = await initializeTracer(); + const currentSessionId = tracer.sessionId; // Capture session ID + console.log(`Initialized tracer with session ID: ${currentSessionId}`); + + // Define the user properties object (from docs snippet) + const userProps: UserProperties = { + user_id: "12345", + user_email: "user@example.com", + user_properties: { + is_premium: true, + subscription_plan: "pro", + last_login: "2024-01-01T12:00:00Z" + } + }; + + try { + // --- Main Execution Logic (from docs snippet) --- + // Wrap the execution logic in tracer.trace() + // Note: Even if only enriching, trace() creates the session context and a span + console.log("Enriching session within tracer.trace()..."); + await tracer.trace(async () => { + // Enrich the session with user properties using standalone function + enrichSession({ + userProperties: userProps // Note: property name is userProperties + }); + console.log("Trace session enriched with user properties."); + }); + console.log("Traced execution (enrichment) finished."); + + + // Ensure data is sent before verification (copied from previous examples) + console.log("Waiting for data propagation after flush..."); + await tracer.flush(); + console.log("Tracer flushed."); + await new Promise(resolve => setTimeout(resolve, 15000)); // Wait + + // --- Verification Logic (Adapted from new_setting_user_properties.ts) --- + console.log(`Starting verification for session ID: ${currentSessionId}`); + const sdk = new HoneyHive({ + bearerAuth: process.env.HH_API_KEY!, + serverURL: process.env.HH_API_URL + }); + + console.log(`Fetching events for session ID: ${currentSessionId}`); + const res = await sdk.events.getEvents({ + project: process.env.HH_PROJECT!, + filters: [ + { + field: "session_id", + value: currentSessionId, + operator: Operator.Is + } + ], + }); + + // Assertions + assert(res.events, `Events response is undefined for session ${currentSessionId}`); + // Expecting Session Start + the span created by tracer.trace + assert(res.events.length >= 2, `Expected at least 2 events for session ${currentSessionId}, found ${res.events.length}`); + console.log(`Found ${res.events.length} events.`); + + // Find the session start event + const sessionEvent = res.events.find((e: Event) => e.eventType === 'session'); + assert(sessionEvent, "Session start event not found"); + console.log(`Found session event: ${sessionEvent.eventId}`); + assert(sessionEvent.userProperties, "User properties not found in session start event"); + console.log("Session event user properties:", sessionEvent.userProperties); + + // Verify the user properties using deepStrictEqual against the defined userProps object + assert.deepStrictEqual(sessionEvent.userProperties, userProps, "User properties do not match"); + console.log("User properties verified successfully."); + // --- End Verification Logic --- + + console.log('Test finished successfully.'); + return true; // Return true only if both execution and verification succeed + + } catch (error) { + console.error(`Error during traced execution or verification for session ${currentSessionId}:`, error); + return false; // Indicate failure + } +} + +export { main }; \ No newline at end of file diff --git a/tests/integration/setting_user_properties_legacy.ts b/tests/integration/setting_user_properties_legacy.ts new file mode 100644 index 0000000..bfae79b --- /dev/null +++ b/tests/integration/setting_user_properties_legacy.ts @@ -0,0 +1,105 @@ +import { HoneyHiveTracer, HoneyHive } from "honeyhive"; +import assert from "assert"; +import { Operator, Event } from "honeyhive/models/components/index.js"; + +// Define a type for the user properties structure if desired, or use any +interface CustomUserProperties { + is_premium: boolean; + subscription_plan: string; + last_login: string; +} + +interface UserProperties { + user_id: string; + user_email: string; + user_properties: CustomUserProperties; +} + +async function main() { + // Environment variable checks + if (!process.env.HH_API_KEY) { + throw new Error("HH_API_KEY environment variable is not set."); + } + if (!process.env.HH_PROJECT) { + throw new Error("HH_PROJECT environment variable is not set."); + } + const serverURL = process.env.HH_API_URL; // Optional + + const tracer = await HoneyHiveTracer.init({ + apiKey: process.env.HH_API_KEY, + project: process.env.HH_PROJECT, + sessionName: "setting-user-props-test", // Distinct session name + serverUrl: serverURL + }); + + const currentSessionId = tracer.sessionId; + console.log(`Initialized tracer with session ID: ${currentSessionId}`); + + // Define the user properties object + const userProps: UserProperties = { + user_id: "12345", + user_email: "user@example.com", + user_properties: { + is_premium: true, + subscription_plan: "pro", + last_login: "2024-01-01T12:00:00Z" + } + }; + + // Enrich the session with user properties + console.log("Enriching session with user properties..."); + tracer.enrichSession({ + userProperties: userProps // Correct property name + }); + console.log("Session enriched."); + + // Wait for data propagation + console.log("Waiting for data propagation..."); + await new Promise(resolve => setTimeout(resolve, 15000)); + + // Initialize SDK + const sdk = new HoneyHive({ + bearerAuth: process.env.HH_API_KEY, + serverURL: serverURL + }); + + // Fetch events for the session + console.log(`Fetching events for session ID: ${currentSessionId}`); + try { + const res = await sdk.events.getEvents({ + project: process.env.HH_PROJECT, + filters: [ + { + field: "session_id", + value: currentSessionId, + operator: Operator.Is + } + ], + }); + + // Assertions + assert(res.events, `Events response is undefined for session ${currentSessionId}`); + // Expecting at least 1 event: Session Start + assert(res.events.length >= 1, `Expected at least 1 event for session ${currentSessionId}, found ${res.events.length}`); + console.log(`Found ${res.events.length} events.`); + + // Find the session start event + const sessionEvent = res.events.find((e: Event) => e.eventType === 'session'); + assert(sessionEvent, "Session start event not found"); + console.log(`Found session event: ${sessionEvent.eventId}`); + assert(sessionEvent.userProperties, "User properties not found in session start event"); + console.log("Session event user properties:", sessionEvent.userProperties); + + // Verify the user properties using deepStrictEqual for object comparison + assert.deepStrictEqual(sessionEvent.userProperties, userProps, "User properties do not match"); + console.log("User properties verified successfully."); + + } catch (error) { + console.error(`Error fetching or verifying events for session ${currentSessionId}:`, error); + throw error; // Re-throw the error to fail the test + } + + console.log('Test finished successfully.'); +} + +export { main }; \ No newline at end of file diff --git a/tests/integration/tracer_ref.ts b/tests/integration/tracer_ref.ts new file mode 100644 index 0000000..a81efe7 --- /dev/null +++ b/tests/integration/tracer_ref.ts @@ -0,0 +1,99 @@ +import { HoneyHiveTracer, HoneyHive } from "honeyhive"; +import assert from "assert"; +import { Operator, Event } from "honeyhive/models/components/index.js"; + +async function main() { + // Environment variable checks + if (!process.env.HH_API_KEY) { + throw new Error("HH_API_KEY environment variable is not set."); + } + if (!process.env.HH_PROJECT) { + throw new Error("HH_PROJECT environment variable is not set."); + } + const serverURL = process.env.HH_API_URL; // Optional + + // Initialize tracer + const tracer = await HoneyHiveTracer.init({ + apiKey: process.env.HH_API_KEY, + project: process.env.HH_PROJECT, + sessionName: "Tracer Ref Test Session", // Using name from Python example + source: "source_identifier", // Using source from Python example + serverUrl: serverURL + }); + + const currentSessionId = tracer.sessionId; + console.log(`Initialized tracer with session ID: ${currentSessionId}`); + + // Define final enrichment objects + const finalFeedback = { 'some_domain_expert': "Final feedback" }; + const finalMetrics = { "final_metric": 1.0 }; + const finalMetadata = { "final_key": "final_value" }; + + // Enrich the session simultaneously + console.log("Enriching session with final feedback, metrics, and metadata..."); + tracer.enrichSession({ + feedback: finalFeedback, + metrics: finalMetrics, + metadata: finalMetadata + }); + console.log("Session enriched."); + + // Wait for data propagation + console.log("Waiting for data propagation..."); + await new Promise(resolve => setTimeout(resolve, 10000)); + + // Initialize SDK + const sdk = new HoneyHive({ + bearerAuth: process.env.HH_API_KEY, + serverURL: serverURL + }); + + // Fetch events for the session + console.log(`Fetching events for session ID: ${currentSessionId}`); + try { + const res = await sdk.events.getEvents({ + project: process.env.HH_PROJECT, + filters: [ + { + field: "session_id", + value: currentSessionId, + operator: Operator.Is + } + ], + }); + + // Assertions + assert(res.events, `Events response is undefined for session ${currentSessionId}`); + // Expecting at least 1 event: Session Start + assert(res.events.length >= 1, `Expected at least 1 event for session ${currentSessionId}, found ${res.events.length}`); + console.log(`Found ${res.events.length} events.`); + + // Find the session start event + const sessionEvent = res.events.find((e: Event) => e.eventType === 'session'); + assert(sessionEvent, "Session start event not found"); + console.log(`Found session event: ${sessionEvent.eventId}`); + + // Verify the final feedback, metrics, and metadata + assert(sessionEvent.feedback, "Feedback not found in session event"); + assert(sessionEvent.metrics, "Metrics not found in session event"); + assert(sessionEvent.metadata, "Metadata not found in session event"); + + console.log("Session event feedback:", sessionEvent.feedback); + console.log("Session event metrics:", sessionEvent.metrics); + console.log("Session event metadata:", sessionEvent.metadata); + + assert.deepStrictEqual(sessionEvent.feedback, finalFeedback, "Feedback does not match"); + assert(sessionEvent.metrics.final_metric === finalMetrics.final_metric, `Metric 'final_metric' does not match`); + assert(sessionEvent.metadata.final_key === finalMetadata.final_key, `Metadata 'final_key' does not match`); + + console.log("Feedback, metrics, and metadata verified successfully."); + + } catch (error) { + console.error(`Error fetching or verifying events for session ${currentSessionId}:`, error); + throw error; // Re-throw the error to fail the test + } + + console.log('Test finished successfully.'); +} + +export { main }; \ No newline at end of file