Skip to content

Commit 4879552

Browse files
andreiborzaandreiborza
committed
chore(ci): Unflake gen_ai tests
Instead of asserting gen_ai spans order, we follow the common streaming approach of finding the spans by name and asserting existence. Closes: #20812
1 parent ad1f373 commit 4879552

14 files changed

Lines changed: 3481 additions & 2366 deletions

File tree

‎dev-packages/cloudflare-integration-tests/suites/tracing/google-genai/test.ts‎

Lines changed: 65 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -29,52 +29,83 @@ it('traces Google GenAI chat creation and message sending', async ({ signal }) =
2929
const container = envelope[1]?.[1]?.[1] as any;
3030
expect(container).toBeDefined();
3131
expect(container.items).toHaveLength(3);
32-
const [firstSpan, secondSpan, thirdSpan] = container.items;
32+
expect(container.items.map(span => span.name).sort()).toEqual([
33+
'chat gemini-1.5-pro',
34+
'embeddings text-embedding-004',
35+
'generate_content gemini-1.5-flash',
36+
]);
3337

34-
// [0] chat gemini-1.5-pro
35-
expect(firstSpan!.name).toBe('chat gemini-1.5-pro');
36-
expect(firstSpan!.status).toBe('ok');
37-
expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' });
38-
expect(firstSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' });
39-
expect(firstSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.google_genai' });
40-
expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'google_genai' });
41-
expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({
38+
const chatSpan = container.items.find(span => span.name === 'chat gemini-1.5-pro');
39+
expect(chatSpan).toBeDefined();
40+
expect(chatSpan!.status).toBe('ok');
41+
expect(chatSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' });
42+
expect(chatSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' });
43+
expect(chatSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.google_genai' });
44+
expect(chatSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'google_genai' });
45+
expect(chatSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({
4246
type: 'string',
4347
value: 'gemini-1.5-pro',
4448
});
45-
expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 8 });
46-
expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 12 });
47-
expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 20 });
49+
expect(chatSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 8 });
50+
expect(chatSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 12 });
51+
expect(chatSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 20 });
4852

49-
// [1] generate_content gemini-1.5-flash
50-
expect(secondSpan!.name).toBe('generate_content gemini-1.5-flash');
51-
expect(secondSpan!.status).toBe('ok');
52-
expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({
53+
const generateContentSpan = container.items.find(span => span.name === 'generate_content gemini-1.5-flash');
54+
expect(generateContentSpan).toBeDefined();
55+
expect(generateContentSpan!.status).toBe('ok');
56+
expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({
5357
type: 'string',
5458
value: 'generate_content',
5559
});
56-
expect(secondSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.generate_content' });
57-
expect(secondSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.google_genai' });
58-
expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'google_genai' });
59-
expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({
60+
expect(generateContentSpan!.attributes['sentry.op']).toEqual({
61+
type: 'string',
62+
value: 'gen_ai.generate_content',
63+
});
64+
expect(generateContentSpan!.attributes['sentry.origin']).toEqual({
65+
type: 'string',
66+
value: 'auto.ai.google_genai',
67+
});
68+
expect(generateContentSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({
69+
type: 'string',
70+
value: 'google_genai',
71+
});
72+
expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({
6073
type: 'string',
6174
value: 'gemini-1.5-flash',
6275
});
63-
expect(secondSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]).toEqual({ type: 'double', value: 0.7 });
64-
expect(secondSpan!.attributes[GEN_AI_REQUEST_TOP_P_ATTRIBUTE]).toEqual({ type: 'double', value: 0.9 });
65-
expect(secondSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 100 });
66-
expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 8 });
67-
expect(secondSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 12 });
68-
expect(secondSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 20 });
76+
expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]).toEqual({
77+
type: 'double',
78+
value: 0.7,
79+
});
80+
expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TOP_P_ATTRIBUTE]).toEqual({ type: 'double', value: 0.9 });
81+
expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]).toEqual({
82+
type: 'integer',
83+
value: 100,
84+
});
85+
expect(generateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({
86+
type: 'integer',
87+
value: 8,
88+
});
89+
expect(generateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({
90+
type: 'integer',
91+
value: 12,
92+
});
93+
expect(generateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({
94+
type: 'integer',
95+
value: 20,
96+
});
6997

70-
// [2] embeddings text-embedding-004
71-
expect(thirdSpan!.name).toBe('embeddings text-embedding-004');
72-
expect(thirdSpan!.status).toBe('ok');
73-
expect(thirdSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'embeddings' });
74-
expect(thirdSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.embeddings' });
75-
expect(thirdSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.google_genai' });
76-
expect(thirdSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'google_genai' });
77-
expect(thirdSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({
98+
const embeddingsSpan = container.items.find(span => span.name === 'embeddings text-embedding-004');
99+
expect(embeddingsSpan).toBeDefined();
100+
expect(embeddingsSpan!.status).toBe('ok');
101+
expect(embeddingsSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({
102+
type: 'string',
103+
value: 'embeddings',
104+
});
105+
expect(embeddingsSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.embeddings' });
106+
expect(embeddingsSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.google_genai' });
107+
expect(embeddingsSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'google_genai' });
108+
expect(embeddingsSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({
78109
type: 'string',
79110
value: 'text-embedding-004',
80111
});

‎dev-packages/cloudflare-integration-tests/suites/tracing/langchain/test.ts‎

Lines changed: 30 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -29,38 +29,42 @@ it('traces langchain chat model, chain, and tool invocations', async ({ signal }
2929
const container = envelope[1]?.[1]?.[1] as any;
3030
expect(container).toBeDefined();
3131
expect(container.items).toHaveLength(3);
32-
const [firstSpan, secondSpan, thirdSpan] = container.items;
32+
expect(container.items.map(span => span.name).sort()).toEqual([
33+
'chain my_test_chain',
34+
'chat claude-3-5-sonnet-20241022',
35+
'execute_tool search_tool',
36+
]);
3337

34-
// [0] chat claude-3-5-sonnet-20241022
35-
expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022');
36-
expect(firstSpan!.status).toBe('ok');
37-
expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' });
38-
expect(firstSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' });
39-
expect(firstSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.langchain' });
40-
expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'anthropic' });
41-
expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({
38+
const chatSpan = container.items.find(span => span.name === 'chat claude-3-5-sonnet-20241022');
39+
expect(chatSpan).toBeDefined();
40+
expect(chatSpan!.status).toBe('ok');
41+
expect(chatSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' });
42+
expect(chatSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' });
43+
expect(chatSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.langchain' });
44+
expect(chatSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'anthropic' });
45+
expect(chatSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({
4246
type: 'string',
4347
value: 'claude-3-5-sonnet-20241022',
4448
});
45-
expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]).toEqual({ type: 'double', value: 0.7 });
46-
expect(firstSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 100 });
47-
expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 10 });
48-
expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 15 });
49-
expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 25 });
49+
expect(chatSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]).toEqual({ type: 'double', value: 0.7 });
50+
expect(chatSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 100 });
51+
expect(chatSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 10 });
52+
expect(chatSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 15 });
53+
expect(chatSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 25 });
5054

51-
// [1] chain my_test_chain
52-
expect(secondSpan!.name).toBe('chain my_test_chain');
53-
expect(secondSpan!.status).toBe('ok');
54-
expect(secondSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.langchain' });
55-
expect(secondSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.invoke_agent' });
56-
expect(secondSpan!.attributes['langchain.chain.name']).toEqual({ type: 'string', value: 'my_test_chain' });
55+
const chainSpan = container.items.find(span => span.name === 'chain my_test_chain');
56+
expect(chainSpan).toBeDefined();
57+
expect(chainSpan!.status).toBe('ok');
58+
expect(chainSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.langchain' });
59+
expect(chainSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.invoke_agent' });
60+
expect(chainSpan!.attributes['langchain.chain.name']).toEqual({ type: 'string', value: 'my_test_chain' });
5761

58-
// [2] execute_tool search_tool
59-
expect(thirdSpan!.name).toBe('execute_tool search_tool');
60-
expect(thirdSpan!.status).toBe('ok');
61-
expect(thirdSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.langchain' });
62-
expect(thirdSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.execute_tool' });
63-
expect(thirdSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'search_tool' });
62+
const toolSpan = container.items.find(span => span.name === 'execute_tool search_tool');
63+
expect(toolSpan).toBeDefined();
64+
expect(toolSpan!.status).toBe('ok');
65+
expect(toolSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.langchain' });
66+
expect(toolSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.execute_tool' });
67+
expect(toolSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'search_tool' });
6468
})
6569
.start(signal);
6670
await runner.makeRequest('get', '/');

‎dev-packages/cloudflare-integration-tests/suites/tracing/langgraph/test.ts‎

Lines changed: 36 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -29,47 +29,62 @@ it('traces langgraph compile and invoke operations', async ({ signal }) => {
2929
expect(container).toBeDefined();
3030

3131
expect(container.items).toHaveLength(2);
32-
const [firstSpan, secondSpan] = container.items;
32+
expect(container.items.map(span => span.name).sort()).toEqual([
33+
'create_agent weather_assistant',
34+
'invoke_agent weather_assistant',
35+
]);
3336

34-
// [0] create_agent weather_assistant
35-
expect(firstSpan!.name).toBe('create_agent weather_assistant');
36-
expect(firstSpan!.status).toBe('ok');
37-
expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({
37+
const createAgentSpan = container.items.find(span => span.name === 'create_agent weather_assistant');
38+
expect(createAgentSpan).toBeDefined();
39+
expect(createAgentSpan!.status).toBe('ok');
40+
expect(createAgentSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({
3841
type: 'string',
3942
value: 'create_agent',
4043
});
41-
expect(firstSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.create_agent' });
42-
expect(firstSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.langgraph' });
43-
expect(firstSpan!.attributes[GEN_AI_AGENT_NAME_ATTRIBUTE]).toEqual({
44+
expect(createAgentSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.create_agent' });
45+
expect(createAgentSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.langgraph' });
46+
expect(createAgentSpan!.attributes[GEN_AI_AGENT_NAME_ATTRIBUTE]).toEqual({
4447
type: 'string',
4548
value: 'weather_assistant',
4649
});
4750

48-
// [1] invoke_agent weather_assistant
49-
expect(secondSpan!.name).toBe('invoke_agent weather_assistant');
50-
expect(secondSpan!.status).toBe('ok');
51-
expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({
51+
const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent weather_assistant');
52+
expect(invokeAgentSpan).toBeDefined();
53+
expect(invokeAgentSpan!.status).toBe('ok');
54+
expect(invokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({
5255
type: 'string',
5356
value: 'invoke_agent',
5457
});
55-
expect(secondSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.invoke_agent' });
56-
expect(secondSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.langgraph' });
57-
expect(secondSpan!.attributes[GEN_AI_AGENT_NAME_ATTRIBUTE]).toEqual({
58+
expect(invokeAgentSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.invoke_agent' });
59+
expect(invokeAgentSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.langgraph' });
60+
expect(invokeAgentSpan!.attributes[GEN_AI_AGENT_NAME_ATTRIBUTE]).toEqual({
5861
type: 'string',
5962
value: 'weather_assistant',
6063
});
61-
expect(secondSpan!.attributes[GEN_AI_PIPELINE_NAME_ATTRIBUTE]).toEqual({
64+
expect(invokeAgentSpan!.attributes[GEN_AI_PIPELINE_NAME_ATTRIBUTE]).toEqual({
6265
type: 'string',
6366
value: 'weather_assistant',
6467
});
65-
expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toEqual({
68+
expect(invokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toEqual({
6669
type: 'string',
6770
value: '[{"role":"user","content":"What is the weather in SF?"}]',
6871
});
69-
expect(secondSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'mock-model' });
70-
expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 20 });
71-
expect(secondSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 10 });
72-
expect(secondSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 30 });
72+
expect(invokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({
73+
type: 'string',
74+
value: 'mock-model',
75+
});
76+
expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({
77+
type: 'integer',
78+
value: 20,
79+
});
80+
expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({
81+
type: 'integer',
82+
value: 10,
83+
});
84+
expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({
85+
type: 'integer',
86+
value: 30,
87+
});
7388
})
7489
.start(signal);
7590
await runner.makeRequest('get', '/');

0 commit comments

Comments
 (0)