integration tests - docs code blocks

tests/integration/client_side_evals.ts

@@ -0,0 +1,81 @@
+import { HoneyHiveTracer, HoneyHive } from "honeyhive";
+import assert from "assert";
+import { Operator } from "honeyhive/models/components/index.js";
+
+async function main() {
+    if (!process.env.HH_API_KEY) {
+        throw new Error("HH_API_KEY environment variable is not set.");
+    }
+    if (!process.env.HH_PROJECT) {
+        throw new Error("HH_PROJECT environment variable is not set.");
+    }
+
+    const tracer = await HoneyHiveTracer.init({
+      apiKey: process.env.HH_API_KEY,
+      project: process.env.HH_PROJECT,
+      serverUrl: process.env.HH_API_URL
+    });
+
+    const currentSessionId = tracer.sessionId;
+
+    tracer.enrichSession({
+      metrics: {
+        "json_validated": true,
+        "num_actions": 10,
+        // any other custom fields and values as you need
+        "step_evals": [
+          {
+            "invalid_grammar": false,
+            "unable_to_locate_UI": true
+          }
+        ],
+      }
+    });
+
+    // Wait 5 seconds for data propagation
+    await new Promise(resolve => setTimeout(resolve, 5000));
+
+    const sdk = new HoneyHive({
+        bearerAuth: process.env.HH_API_KEY,
+        serverURL: process.env.HH_API_URL
+    });
+
+    console.log(`Fetching events for session ID: ${currentSessionId}`);
+    try {
+        const res = await sdk.events.getEvents({
+            project: process.env.HH_PROJECT,
+            filters: [
+                {
+                    field: "event_type",
+                    value: "session",
+                    operator: Operator.Is
+                },
+                {
+                    field: "session_id",
+                    value: currentSessionId,
+                    operator: Operator.Is
+                }
+            ],
+        });
+
+        // Assert that events exist and the first event has the expected metrics
+        assert(res.events && res.events.length > 0, `No events found for session ${currentSessionId}`);
+        const firstEvent = res.events[0];
+        assert(firstEvent.metrics, `Metrics not found for the first event in session ${currentSessionId}`);
+        assert.strictEqual(firstEvent.metrics.json_validated, true, "json_validated metric does not match");
+        assert.strictEqual(firstEvent.metrics.num_actions, 10, "num_actions metric does not match");
+        assert(Array.isArray(firstEvent.metrics.step_evals), "step_evals metric is not an array");
+        assert.strictEqual(firstEvent.metrics.step_evals[0]?.invalid_grammar, false, "step_evals[0].invalid_grammar does not match");
+        assert.strictEqual(firstEvent.metrics.step_evals[0]?.unable_to_locate_UI, true, "step_evals[0].unable_to_locate_UI does not match");
+
+        console.log(`Events for session ${currentSessionId}:`, res.events);
+
+    } catch (error) {
+        console.error(`Error fetching events for session ${currentSessionId}:`, error);
+        throw error; // Re-throw the error to fail the test
+    }
+
+    console.log('Finished fetching events.');
+}
+
+export { main };

tests/integration/experiments_quickstart.ts

@@ -0,0 +1,147 @@
+import { evaluate } from "honeyhive";
+import { OpenAI } from 'openai';
+import { HoneyHive } from "honeyhive";
+import assert from "assert";
+import { Operator } from "honeyhive/models/components/index.js";
+
+const openai = new OpenAI({ apiKey: process.env.OPENAI_KEY });
+
+// Create function to be evaluated
+// input -> parameter to which datapoint or json value will be passed
+export async function functionToEvaluate(input: Record<string, any>, ground_truths: Record<string, any>) {
+
+    try {
+        const response = await openai.chat.completions.create({
+            model: "gpt-4",
+            messages: [
+                {
+                    role: 'system',
+                    content: `You are an expert analyst specializing in ${input.product_type} market trends.`
+                },
+                {
+                    role: 'user',
+                    content: `Could you provide an analysis of the current market performance and consumer reception of ${input.product_type} in ${input.region}? Please include any notable trends or challenges specific to this region.`
+                }
+            ],
+        });
+
+        // Output -> session output
+        return response.choices[0].message;
+    } catch (error) {
+        console.error('Error making GPT-4 call:', error);
+        throw error;
+    }
+}
+
+const dataset = [
+    {
+        "inputs": {
+            "product_type": "electric vehicles",
+            "region": "western europe"   
+        },
+        "ground_truths": {
+            "response": "As of 2023, the electric vehicle (EV) ... ",
+        }
+    },
+    {
+        "inputs": {
+            "product_type": "gaming consoles",
+            "region": "north america"
+        },
+        "ground_truths": {
+            "response": "As of 2023, the gaming console market ... ",
+        }
+    },
+    {
+        "inputs": {
+            "product_type": "smart home devices",
+            "region": "australia and new zealand" 
+        },
+        "ground_truths": {
+            "response": "As of 2023, the market for smart home devices in Australia and New Zealand ... ",
+        }
+    }
+]
+
+// Sample evaluator that returns fixed metrics
+function sampleEvaluator(output: any, input: Record<string, any>, ground_truths: Record<string, any>) {
+    // Code here
+    return {
+        sample_metric: 0.5,
+        sample_metric_2: true
+    };
+}
+
+async function main() {
+
+    if (!process.env.HH_API_KEY) {
+        throw new Error("HH_API_KEY environment variable is not set.");
+    }
+    if (!process.env.HH_PROJECT) {
+        throw new Error("HH_PROJECT environment variable is not set.");
+    }
+    if (!process.env.OPENAI_API_KEY) {
+        throw new Error("OPENAI_API_KEY environment variable is not set.");
+    }
+
+    const result = await  evaluate({
+            function: functionToEvaluate,       // Function to be evaluated
+            apiKey: process.env.HH_API_KEY,
+            project: process.env.HH_PROJECT,
+            name: 'Sample Experiment',
+            dataset: dataset,                        // to be passed for json_list
+            evaluators: [sampleEvaluator],                // to compute client-side metrics on each run
+            serverUrl: process.env.HH_API_URL
+        });
+
+        assert(result, "Evaluate result should not be undefined");
+        const sessionIds = result.sessionIds;
+        assert(sessionIds && sessionIds.length > 0, "sessionIds should be populated");
+        // Wait 5 seconds for data propagation
+        await new Promise(resolve => setTimeout(resolve, 10000));
+        const sdk = new HoneyHive({
+            bearerAuth: process.env.HH_API_KEY,
+        });
+
+
+        for (const sessionId of sessionIds) {
+            console.log(`Fetching events for session ID: ${sessionId}`);
+            try {
+                const res = await sdk.events.getEvents({
+                    project: process.env.HH_PROJECT,
+                    filters: [
+                        {
+                            field: "event_type",
+                            value: "session",
+                            operator: Operator.Is
+                        },
+                        {
+                            field: "session_id",
+                            value: sessionId,
+                            operator: Operator.Is
+                        }
+                    ],
+                });
+
+                // Assert that events exist and the first event has the expected metrics
+                assert(res.events && res.events.length > 0, `No events found for session ${sessionId}`);
+                const firstEvent = res.events[0];
+                assert(firstEvent.metrics, `Metrics not found for the first event in session ${sessionId}`);
+                assert(firstEvent.metrics.sampleEvaluator, `sampleEvaluator metric not found in the first event for session ${sessionId}`);
+                assert.deepStrictEqual(firstEvent.metrics.sampleEvaluator, 
+                    { sample_metric: 0.5, sample_metric_2: true },
+                    `Metrics for sampleEvaluator in session ${sessionId} do not match the expected values.`
+                );
+
+                console.log(`Events for session ${sessionId}:`, res.events);
+
+            } catch (error) {
+                console.error(`Error fetching events for session ${sessionId}:`, error);
+            }
+        }
+
+        console.log('Finished fetching events for all sessions.');
+
+}
+
+export { main };

tests/integration/introduction_quickstart.ts

@@ -0,0 +1,74 @@
+import { HoneyHiveTracer, HoneyHive } from "honeyhive";
+import assert from "assert";
+import { Operator } from "honeyhive/models/components/index.js";
+
+
+async function main() {
+    // Environment variable checks
+    if (!process.env.HH_API_KEY) {
+        throw new Error("HH_API_KEY environment variable is not set.");
+    }
+    if (!process.env.HH_PROJECT) {
+        throw new Error("HH_PROJECT environment variable is not set.");
+    }
+    // Optional variables - provide defaults or handle absence if needed
+    const source = process.env.HH_SOURCE; // Optional
+    const sessionName = process.env.HH_SESSION; // Optional
+    const serverURL = process.env.HH_API_URL; // Optional
+
+    // Initialize tracer
+    const tracer = await HoneyHiveTracer.init({
+      apiKey: process.env.HH_API_KEY,
+      project: process.env.HH_PROJECT,
+      source: source, 
+      sessionName: sessionName, 
+      serverUrl: serverURL // Correct property name is serverUrl
+    });
+
+    const currentSessionId = tracer.sessionId;
+    console.log(`Initialized tracer with session ID: ${currentSessionId}`);
+
+    // Wait for data propagation
+    await new Promise(resolve => setTimeout(resolve, 10000));
+
+    // Initialize SDK
+    const sdk = new HoneyHive({
+        bearerAuth: process.env.HH_API_KEY,
+        serverURL: serverURL // Correct property name is serverURL here
+    });
+
+    // Fetch events for the session
+    console.log(`Fetching events for session ID: ${currentSessionId}`);
+    try {
+        const res = await sdk.events.getEvents({
+            project: process.env.HH_PROJECT,
+            filters: [
+                {
+                    field: "event_type",
+                    value: "session",
+                    operator: Operator.Is
+                },
+                {
+                    field: "session_id", // Filter uses session_id
+                    value: currentSessionId,
+                    operator: Operator.Is
+                }
+            ],
+        });
+
+        // Assertions
+        assert(res.events, `Events response is undefined for session ${currentSessionId}`);
+        assert(res.events.length > 0, `No events found for session ${currentSessionId}`);
+        const firstEvent = res.events[0];
+        assert.strictEqual(firstEvent.sessionId, currentSessionId, `Event session ID ${firstEvent.sessionId} does not match expected ${currentSessionId}`); // Event object uses sessionId
+        console.log(`Successfully verified session start event for session ID: ${currentSessionId}`);
+
+    } catch (error) {
+        console.error(`Error fetching or verifying events for session ${currentSessionId}:`, error);
+        throw error; // Re-throw the error to fail the test
+    }
+
+    console.log('Test finished successfully.');
+}
+
+export { main };