diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5328387..94174698 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,6 +85,16 @@ jobs: - name: Run tests run: npm test + - name: Run integration tests + env: + AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} + AZURE_OPENAI_DEPLOYMENT: ${{ vars.AZURE_OPENAI_DEPLOYMENT }} + AZURE_OPENAI_API_VERSION: ${{ vars.AZURE_OPENAI_API_VERSION }} + ENABLE_OBSERVABILITY: ${{ vars.ENABLE_OBSERVABILITY }} + run: | + npm run test:integration + - name: Pack packages run: npm run pack diff --git a/jest.integration.config.cjs b/jest.integration.config.cjs new file mode 100644 index 00000000..397f2394 --- /dev/null +++ b/jest.integration.config.cjs @@ -0,0 +1,27 @@ +// ------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------------------------ + +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + rootDir: '.', + testMatch: ['**/tests/observability/integration/**/*.test.ts'], + moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json'], + collectCoverageFrom: [ + 'tests/observability/integration/**/*.ts', + '!tests/observability/integration/**/*.d.ts', + '!tests/observability/integration/**/*.test.ts', + ], + coverageDirectory: 'coverage/integration', + setupFilesAfterEnv: ['/tests/observability/integration/setup.ts'], + testTimeout: 60000, + globals: { + 'ts-jest': { + tsconfig: { + module: 'commonjs', + esModuleInterop: true, + }, + }, + }, +}; diff --git a/package.json b/package.json index e47b1691..de3f5875 100644 --- a/package.json +++ b/package.json @@ -8,8 +8,10 @@ "build": "pnpm -r build", "build:watch": "npm run build:watch --workspaces --if-present", "clean": "npm run clean --workspaces --if-present && rimraf node_modules", - "test": "npm run test --workspaces --if-present", + "test": "npm run test --workspaces --if-present -- --testPathIgnorePatterns=/integration/", "test:watch": "npm run test:watch --workspaces --if-present", + "test:integration": "jest --config jest.integration.config.cjs", + "test:integration:watch": "jest --config jest.integration.config.cjs --watch", "lint": "npm run lint --workspaces --if-present", "lint:fix": "npm run lint:fix --workspaces --if-present", "ci": "npm run ci --workspaces --if-present", diff --git a/packages/agents-a365-observability-extensions-openai/jest.config.json b/packages/agents-a365-observability-extensions-openai/jest.config.json index 7e6c5ea9..a1c08051 100644 --- a/packages/agents-a365-observability-extensions-openai/jest.config.json +++ b/packages/agents-a365-observability-extensions-openai/jest.config.json @@ -1,7 +1,7 @@ { "preset": "ts-jest", "testEnvironment": "node", - "roots": ["/src", "/../../tests/agents-a365-observability-extensions-openai"], + "roots": ["/src", "/../../tests/observability/extension/openai"], "testMatch": [ "**/__tests__/**/*.ts", "**/?(*.)+(spec|test).ts" diff --git a/packages/agents-a365-observability-extensions-openai/src/Constants.ts b/packages/agents-a365-observability-extensions-openai/src/Constants.ts index 6ed2ba1a..8d11ba28 100644 --- a/packages/agents-a365-observability-extensions-openai/src/Constants.ts +++ b/packages/agents-a365-observability-extensions-openai/src/Constants.ts @@ -34,3 +34,4 @@ export const GEN_AI_GRAPH_NODE_PARENT_ID = 'graph_node_parent_id'; export const GEN_AI_REQUEST_CONTENT_KEY = 'gen_ai.request.content'; export const GEN_AI_RESPONSE_CONTENT_KEY = 'gen_ai.response.content'; +export const GEN_AI_EXECUTION_PAYLOAD_KEY = 'gen_ai.execution.payload'; diff --git a/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceProcessor.ts b/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceProcessor.ts index 405b877a..6967ba46 100644 --- a/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceProcessor.ts +++ b/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceProcessor.ts @@ -43,6 +43,8 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor { ['mcp_tools' + Constants.GEN_AI_REQUEST_CONTENT_KEY, OpenTelemetryConstants.GEN_AI_TOOL_ARGS_KEY], ['function' + Constants.GEN_AI_RESPONSE_CONTENT_KEY, OpenTelemetryConstants.GEN_AI_EVENT_CONTENT], ['function' + Constants.GEN_AI_REQUEST_CONTENT_KEY, OpenTelemetryConstants.GEN_AI_TOOL_ARGS_KEY], + ['generation' + Constants.GEN_AI_RESPONSE_CONTENT_KEY, OpenTelemetryConstants.GEN_AI_OUTPUT_MESSAGES_KEY], + ['generation' + Constants.GEN_AI_REQUEST_CONTENT_KEY, OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY], ]); constructor(tracer: OtelTracer) { @@ -119,6 +121,10 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor { parentContext ); + if (!parentSpan) { + this.rootSpans.set(traceId, otelSpan); + } + // Store span and activate context this.otelSpans.set(spanId, otelSpan); this.spanNames.set(otelSpan, spanName); @@ -269,8 +275,11 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor { private processGenerationSpanData(otelSpan: OtelSpan, data: SpanData, traceId: string): void { const attrs = Utils.getAttributesFromGenerationSpanData(data); Object.entries(attrs).forEach(([key, value]) => { - if (value !== null && value !== undefined) { - otelSpan.setAttribute(key, value as string | number | boolean); + const shouldExcludeKey = key === OpenTelemetryConstants.GEN_AI_EXECUTION_TYPE_KEY + || key === Constants.GEN_AI_EXECUTION_PAYLOAD_KEY; + if (value !== null && value !== undefined && !shouldExcludeKey) { + const newKey = this.getNewKey(data.type, key); + otelSpan.setAttribute(newKey || key, value as string | number | boolean); } }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 76e391ed..2e789262 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -435,6 +435,9 @@ importers: '@openai/agents': specifier: ^0.1.5 version: 0.1.11(@cfworker/json-schema@4.1.1)(ws@8.18.3)(zod@4.1.12) + '@openai/agents-openai': + specifier: '*' + version: 0.1.11(@cfworker/json-schema@4.1.1)(ws@8.18.3)(zod@4.1.12) '@opentelemetry/api': specifier: ^1.9.0 version: 1.9.0 @@ -453,6 +456,12 @@ importers: '@opentelemetry/semantic-conventions': specifier: ^1.37.0 version: 1.37.0 + dotenv: + specifier: ^17.2.2 + version: 17.2.3 + openai: + specifier: ^4.0.0 + version: 4.104.0(ws@8.18.3)(zod@4.1.12) devDependencies: '@babel/preset-typescript': specifier: ^7.27.1 @@ -564,6 +573,9 @@ importers: specifier: ^5.1.0 version: 5.1.0 devDependencies: + '@jest/globals': + specifier: ^29.7.0 + version: 29.7.0 '@microsoft/m365agentsplayground': specifier: ^0.2.18 version: 0.2.20 @@ -582,12 +594,18 @@ importers: eslint: specifier: ^8.57.0 version: 8.57.1 + jest: + specifier: ^29.7.0 + version: 29.7.0(@types/node@20.19.24)(ts-node@10.9.2(@types/node@20.19.24)(typescript@5.9.3)) nodemon: specifier: ^3.1.10 version: 3.1.10 rimraf: specifier: ^6.0.0 version: 6.1.0 + ts-jest: + specifier: ^29.2.0 + version: 29.4.5(@babel/core@7.28.5)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.28.5))(jest-util@29.7.0)(jest@29.7.0(@types/node@20.19.24)(ts-node@10.9.2(@types/node@20.19.24)(typescript@5.9.3)))(typescript@5.9.3) ts-node: specifier: ^10.9.2 version: 10.9.2(@types/node@20.19.24)(typescript@5.9.3) @@ -1509,6 +1527,9 @@ packages: '@types/ms@2.1.0': resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==} + '@types/node-fetch@2.6.13': + resolution: {integrity: sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==} + '@types/node@20.19.24': resolution: {integrity: sha512-FE5u0ezmi6y9OZEzlJfg37mqqf6ZDSF2V/NLjUyGrR9uTZ7Sb9F7bLNZ03S4XVUNRWGA7Ck4c1kK+YnuWjl+DA==} @@ -1613,6 +1634,10 @@ packages: '@ungap/structured-clone@1.3.0': resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==} + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + accepts@2.0.0: resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} engines: {node: '>= 0.6'} @@ -1640,6 +1665,10 @@ packages: resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} engines: {node: '>= 14'} + agentkeepalive@4.6.0: + resolution: {integrity: sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==} + engines: {node: '>= 8.0.0'} + ajv-formats@3.0.1: resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==} peerDependencies: @@ -2064,6 +2093,10 @@ packages: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + eventemitter3@4.0.7: resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} @@ -2162,10 +2195,17 @@ packages: resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} engines: {node: '>=14'} + form-data-encoder@1.7.2: + resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} + form-data@4.0.4: resolution: {integrity: sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==} engines: {node: '>= 6'} + formdata-node@4.4.1: + resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} + engines: {node: '>= 12.20'} + forwarded@0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} @@ -2288,6 +2328,9 @@ packages: resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==} engines: {node: '>=10.17.0'} + humanize-ms@1.2.1: + resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} + iconv-lite@0.6.3: resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} engines: {node: '>=0.10.0'} @@ -2793,6 +2836,20 @@ packages: resolution: {integrity: sha512-HbwT9OA/XBLgtUqWWqpiA3glZRHV7kS/gjizguGElVXYzCNvhF7hHOEt1nuQ6wsGNAYLmwAMtEQcnHQqSMbItQ==} hasBin: true + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead + + node-fetch@2.7.0: + resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + node-int64@0.4.0: resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} @@ -2839,6 +2896,18 @@ packages: resolution: {integrity: sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==} engines: {node: '>=18'} + openai@4.104.0: + resolution: {integrity: sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA==} + hasBin: true + peerDependencies: + ws: ^8.18.0 + zod: ^4.1.12 + peerDependenciesMeta: + ws: + optional: true + zod: + optional: true + openai@5.23.2: resolution: {integrity: sha512-MQBzmTulj+MM5O8SKEk/gL8a7s5mktS9zUtAkU257WjvobGc9nKcBuVwjyEEcb9SI8a8Y2G/mzn3vm9n1Jlleg==} hasBin: true @@ -3226,6 +3295,9 @@ packages: resolution: {integrity: sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==} hasBin: true + tr46@0.0.3: + resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + ts-api-utils@1.4.3: resolution: {integrity: sha512-i3eMG77UTMD0hZhgRS562pv83RC6ukSAC2GMNWc+9dieh/+jDM5u5YG+NHX6VNDRHQcHwmsTHctP9LhbC3WxVw==} engines: {node: '>=16'} @@ -3347,6 +3419,16 @@ packages: walker@1.0.8: resolution: {integrity: sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==} + web-streams-polyfill@4.0.0-beta.3: + resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} + engines: {node: '>= 14'} + + webidl-conversions@3.0.1: + resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + + whatwg-url@5.0.0: + resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + which@2.0.2: resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} engines: {node: '>= 8'} @@ -4703,6 +4785,11 @@ snapshots: '@types/ms@2.1.0': {} + '@types/node-fetch@2.6.13': + dependencies: + '@types/node': 20.19.24 + form-data: 4.0.4 + '@types/node@20.19.24': dependencies: undici-types: 6.21.0 @@ -4840,6 +4927,10 @@ snapshots: '@ungap/structured-clone@1.3.0': {} + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + accepts@2.0.0: dependencies: mime-types: 3.0.1 @@ -4861,6 +4952,10 @@ snapshots: agent-base@7.1.4: {} + agentkeepalive@4.6.0: + dependencies: + humanize-ms: 1.2.1 + ajv-formats@3.0.1(ajv@8.17.1): optionalDependencies: ajv: 8.17.1 @@ -5309,6 +5404,8 @@ snapshots: etag@1.8.1: {} + event-target-shim@5.0.1: {} + eventemitter3@4.0.7: {} eventsource-parser@3.0.6: {} @@ -5446,6 +5543,8 @@ snapshots: cross-spawn: 7.0.6 signal-exit: 4.1.0 + form-data-encoder@1.7.2: {} + form-data@4.0.4: dependencies: asynckit: 0.4.0 @@ -5454,6 +5553,11 @@ snapshots: hasown: 2.0.2 mime-types: 2.1.35 + formdata-node@4.4.1: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 4.0.0-beta.3 + forwarded@0.2.0: {} fresh@2.0.0: {} @@ -5585,6 +5689,10 @@ snapshots: human-signals@2.1.0: {} + humanize-ms@1.2.1: + dependencies: + ms: 2.1.3 + iconv-lite@0.6.3: dependencies: safer-buffer: 2.1.2 @@ -6233,6 +6341,12 @@ snapshots: nerdbank-gitversioning@3.8.118: {} + node-domexception@1.0.0: {} + + node-fetch@2.7.0: + dependencies: + whatwg-url: 5.0.0 + node-int64@0.4.0: {} node-releases@2.0.27: {} @@ -6281,6 +6395,21 @@ snapshots: is-inside-container: 1.0.0 wsl-utils: 0.1.0 + openai@4.104.0(ws@8.18.3)(zod@4.1.12): + dependencies: + '@types/node': 20.19.24 + '@types/node-fetch': 2.6.13 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + optionalDependencies: + ws: 8.18.3 + zod: 4.1.12 + transitivePeerDependencies: + - encoding + openai@5.23.2(ws@8.18.3)(zod@4.1.12): optionalDependencies: ws: 8.18.3 @@ -6651,6 +6780,8 @@ snapshots: touch@3.1.1: {} + tr46@0.0.3: {} + ts-api-utils@1.4.3(typescript@5.9.3): dependencies: typescript: 5.9.3 @@ -6750,6 +6881,15 @@ snapshots: dependencies: makeerror: 1.0.12 + web-streams-polyfill@4.0.0-beta.3: {} + + webidl-conversions@3.0.1: {} + + whatwg-url@5.0.0: + dependencies: + tr46: 0.0.3 + webidl-conversions: 3.0.1 + which@2.0.2: dependencies: isexe: 2.0.0 diff --git a/tests-agent/openai-agent-auto-instrument-sample/package.json b/tests-agent/openai-agent-auto-instrument-sample/package.json index 0175cc2c..12da99e9 100644 --- a/tests-agent/openai-agent-auto-instrument-sample/package.json +++ b/tests-agent/openai-agent-auto-instrument-sample/package.json @@ -8,6 +8,9 @@ "start": "node dist/index.js", "dev": "nodemon --watch src/*.ts --exec ts-node src/index.ts", "test-tool": "agentsplayground", + "test:integration": "jest --config ../../jest.integration.config.js", + "test:integration:watch": "jest --config ../../jest.integration.config.js --watch", + "test:setup": "ts-node ../../tests/integration/setup.ts", "lint": "eslint src/**/*.ts", "lint:fix": "eslint src/**/*.ts --fix", "install:clean": "npm run clean && npm install", @@ -36,14 +39,17 @@ "express": "^5.1.0" }, "devDependencies": { + "@jest/globals": "^29.7.0", "@microsoft/m365agentsplayground": "^0.2.18", "@types/express": "^5.0.0", "@types/node": "^18.0.0", "@typescript-eslint/eslint-plugin": "^6.0.0", "@typescript-eslint/parser": "^6.0.0", "eslint": "^8.0.0", + "jest": "^29.7.0", "nodemon": "^3.1.10", "rimraf": "^5.0.0", + "ts-jest": "^29.1.0", "ts-node": "^10.9.2", "typescript": "^5.0.0" }, diff --git a/tests/BaggageBuilder.test.ts b/tests/observability/core/BaggageBuilder.test.ts similarity index 100% rename from tests/BaggageBuilder.test.ts rename to tests/observability/core/BaggageBuilder.test.ts diff --git a/tests/SpanProcessor.test.ts b/tests/observability/core/SpanProcessor.test.ts similarity index 100% rename from tests/SpanProcessor.test.ts rename to tests/observability/core/SpanProcessor.test.ts diff --git a/tests/observabilityManager.test.ts b/tests/observability/core/observabilityManager.test.ts similarity index 100% rename from tests/observabilityManager.test.ts rename to tests/observability/core/observabilityManager.test.ts diff --git a/tests/scopes.test.ts b/tests/observability/core/scopes.test.ts similarity index 100% rename from tests/scopes.test.ts rename to tests/observability/core/scopes.test.ts diff --git a/tests/agents-a365-observability-extensions-openai/OpenAIAgentsTraceInstrumentor.test.ts b/tests/observability/extension/openai/OpenAIAgentsTraceInstrumentor.test.ts similarity index 100% rename from tests/agents-a365-observability-extensions-openai/OpenAIAgentsTraceInstrumentor.test.ts rename to tests/observability/extension/openai/OpenAIAgentsTraceInstrumentor.test.ts diff --git a/tests/agents-a365-observability-extensions-openai/OpenAIAgentsTraceProcessor.test.ts b/tests/observability/extension/openai/OpenAIAgentsTraceProcessor.test.ts similarity index 100% rename from tests/agents-a365-observability-extensions-openai/OpenAIAgentsTraceProcessor.test.ts rename to tests/observability/extension/openai/OpenAIAgentsTraceProcessor.test.ts diff --git a/tests/observability/integration/.env.example b/tests/observability/integration/.env.example new file mode 100644 index 00000000..199bd60b --- /dev/null +++ b/tests/observability/integration/.env.example @@ -0,0 +1,14 @@ +# Environment variables for integration tests +# Copy this file to tests/.env and fill in your values + +# Azure OpenAI Configuration (required for integration tests) +AZURE_OPENAI_API_KEY=your-api-key-here +AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com +AZURE_OPENAI_DEPLOYMENT=gpt-4 + +# Test Configuration +TEST_LOG_LEVEL=info + +# Skip tests when Azure OpenAI config is missing (default: true) +# Set to 'false' to run tests anyway (they will fail without valid config) +SKIP_ON_MISSING_CONFIG=false diff --git a/tests/observability/integration/conftest.ts b/tests/observability/integration/conftest.ts new file mode 100644 index 00000000..9e2911e2 --- /dev/null +++ b/tests/observability/integration/conftest.ts @@ -0,0 +1,80 @@ +// ------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------------------------ + +/** + * Test configuration and utilities based on Python conftest.py pattern + */ + +import * as dotenv from 'dotenv'; +import * as path from 'path'; + +// Load .env file if it exists (for local development) +const envFile = path.join(__dirname, '.', '.env'); + +try { + dotenv.config({ path: envFile }); +} catch (error) { + // .env file is optional +} + +/** + * Azure OpenAI configuration interface + */ +export interface AzureOpenAIConfig { + apiKey: string; + endpoint: string; + deployment: string; + apiVersion?: string; +} + +/** + * Agent365 configuration interface + */ +export interface Agent365Config { + tenantId?: string; + correlationId?: string; + agentId?: string; +} + +/** + * Get Azure OpenAI configuration from environment variables + */ +export function getAzureOpenAIConfig(): AzureOpenAIConfig | null { + const apiKey = process.env.AZURE_OPENAI_API_KEY; + const endpoint = process.env.AZURE_OPENAI_ENDPOINT; + const deployment = process.env.AZURE_OPENAI_DEPLOYMENT; + + if (!apiKey || !endpoint || !deployment) { + return null; + } + + return { + apiKey, + endpoint, + deployment, + apiVersion: process.env.AZURE_OPENAI_API_VERSION || '2024-08-01-preview', + }; +} + +/** + * Get Agent365 configuration from environment variables + */ +export function getAgent365Config(): Agent365Config { + return { + tenantId: process.env.AGENT365_TENANT_ID || 'test-tenant-id', + correlationId: process.env.AGENT365_CORRELATION_ID || 'test-correlation-id', + agentId: process.env.AGENT365_AGENT_ID || 'test-agent-id', + }; +} + +/** + * Environment validation helper + */ +export function validateEnvironment(): void { + const azureConfig = getAzureOpenAIConfig(); + + if (!azureConfig) { + throw new Error('Missing required environment variables: AZURE_OPENAI_API_KEY, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_DEPLOYMENT'); + } +} diff --git a/tests/observability/integration/openai-agent-instrument.test.ts b/tests/observability/integration/openai-agent-instrument.test.ts new file mode 100644 index 00000000..30b70366 --- /dev/null +++ b/tests/observability/integration/openai-agent-instrument.test.ts @@ -0,0 +1,402 @@ +// ------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------------------------ + +import { describe, it, expect, beforeAll, afterAll, beforeEach } from "@jest/globals"; +import { getAzureOpenAIConfig, validateEnvironment } from "./conftest"; +import { ReadableSpan } from "@opentelemetry/sdk-trace-base"; +import { ObservabilityManager, Builder, OpenTelemetryConstants } from "@microsoft/agents-a365-observability"; +import { OpenAIAgentsTraceInstrumentor } from "@microsoft/agents-a365-observability-extensions-openai"; +import { Agent, run, tool } from "@openai/agents"; +import { OpenAIChatCompletionsModel } from "@openai/agents-openai"; +import { ObservabilityBuilder } from "@microsoft/agents-a365-observability/dist/esm/ObservabilityBuilder"; +import { AzureOpenAI } from "openai"; + +// Test instrumentation constants +const TEST_INSTRUMENTATION_NAME = "openai-agent-test-instrumentation"; +const TEST_INSTRUMENTATION_VERSION = "1.0.0"; + +describe("OpenAI Trace Processor Integration Tests", () => { + let openAIAgentsTraceInstrumentor: OpenAIAgentsTraceInstrumentor; + let a365Observability: ObservabilityBuilder; + let consoleDirSpy: jest.SpyInstance; + let spans: ReadableSpan[] = []; + + beforeAll(async () => { + validateEnvironment(); + console.log("Setting up OpenAI Trace Processor test suite..."); + + // Also spy on console.dir which ConsoleSpanExporter uses + consoleDirSpy = jest + .spyOn(console, "dir") + .mockImplementation((obj: any) => { + spans.push(obj as ReadableSpan); + }); + + // Configure observability following the sample pattern + a365Observability = ObservabilityManager.configure((builder: Builder) => + builder.withService("OpenAI Agent Instrumentation Sample", "1.0.0"), + ); + + // Initialize OpenAI Agents instrumentation + openAIAgentsTraceInstrumentor = new OpenAIAgentsTraceInstrumentor({ + enabled: true, + tracerName: TEST_INSTRUMENTATION_NAME, + tracerVersion: TEST_INSTRUMENTATION_VERSION, + }); + + // Start observability + a365Observability.start(); + + // Enable instrumentation + openAIAgentsTraceInstrumentor.enable(); + }); + + afterAll(async () => { + console.log("๐Ÿงน Tearing down OpenAI Trace Processor test suite..."); + + // Restore console.log + if (consoleDirSpy) { + consoleDirSpy.mockRestore(); + } + + // Disable instrumentation + if (openAIAgentsTraceInstrumentor) { + openAIAgentsTraceInstrumentor.disable(); + } + + // Shutdown observability + if (a365Observability) { + await a365Observability.shutdown(); + } + + console.log("โœ… OpenAI Trace Processor test suite teardown complete"); + }); + + beforeEach(() => { + // Clear spans for each test + spans = []; + }); + + it("validate agent span and generation span", async () => { + const azureConfig = getAzureOpenAIConfig(); + + if (!azureConfig) { + throw new Error("Azure OpenAI configuration is required"); + } + + try { + let agent: Agent; + + const azureClient = new AzureOpenAI({ + endpoint: azureConfig.endpoint, + deployment: azureConfig.deployment, + apiKey: azureConfig.apiKey, + apiVersion: azureConfig.apiVersion, + }); + + const agentName = "Test Agent"; + agent = new Agent({ + name: agentName, + model: new OpenAIChatCompletionsModel( + azureClient as any, + azureConfig.deployment, + ), + instructions: "You are a helpful assistant.", + }); + + // Run agent with a simple prompt + const prompt = "Say hello!"; + const result = await run(agent, prompt); + + // Wait for spans with timeout (poll until length >= 2 or timeout after 5s) + const startTime = Date.now(); + const timeout = 5000; + while (spans.length < 2 && Date.now() - startTime < timeout) { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + + // Verify we captured spans + expect(spans.length).toBeGreaterThanOrEqual(2); + console.log("Total spans captured:", spans.length); + + // Output all the spans + spans.forEach((span, idx) => { + console.log(`\n--- Span ${idx + 1} of ${spans.length} ---`); + console.log(JSON.stringify(span, null, 2)); + }); + + // Find the generation span + const generationSpan = spans.find((span) => span.name === "generation"); + expect(generationSpan).toBeDefined(); + console.log("Validate generation span"); + if (generationSpan) { + validateInstrumentationScope(generationSpan); + validateSpanProperties(generationSpan); + + // Validate gen_ai attributes + expect( + generationSpan.attributes[ + OpenTelemetryConstants.GEN_AI_OPERATION_NAME_KEY + ], + ).toBe("chat"); + expect( + generationSpan.attributes[OpenTelemetryConstants.GEN_AI_SYSTEM_KEY], + ).toBe("openai"); + expect( + generationSpan.attributes[ + OpenTelemetryConstants.GEN_AI_PROVIDER_NAME_KEY + ], + ).toBe("openai"); + expect( + generationSpan.attributes[ + OpenTelemetryConstants.GEN_AI_REQUEST_MODEL_KEY + ], + ).toBe(azureConfig.deployment); + expect( + generationSpan.attributes[ + OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY + ], + ).toBeDefined(); + expect( + generationSpan.attributes[ + OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY + ], + ).toContain(prompt); + expect( + generationSpan.attributes[ + OpenTelemetryConstants.GEN_AI_OUTPUT_MESSAGES_KEY + ], + ).toBeDefined(); + expect( + generationSpan.attributes[ + OpenTelemetryConstants.GEN_AI_OUTPUT_MESSAGES_KEY + ], + ).toContain("chat.completion"); + + // Validate status + expect(generationSpan.status).toBeDefined(); + expect(generationSpan.status.code).toBe(1); + + console.log("โœ… Generation span validation passed"); + } + + // Find and validate the agent span + const agentSpan = spans.find( + (span) => span.name === `invoke_agent ${agentName}`, + ); + expect(agentSpan).toBeDefined(); + console.log("Validate agent span"); + + if (agentSpan) { + validateInstrumentationScope(agentSpan); + validateSpanProperties(agentSpan); + + // Validate agent-specific attributes + expect( + agentSpan.attributes[ + OpenTelemetryConstants.GEN_AI_OPERATION_NAME_KEY + ], + ).toBe("invoke_agent"); + expect( + agentSpan.attributes[OpenTelemetryConstants.GEN_AI_SYSTEM_KEY], + ).toBe("openai"); + expect( + agentSpan?.attributes[ + OpenTelemetryConstants.CUSTOM_PARENT_SPAN_ID_KEY + ], + ).toBeUndefined(); + + validateParentChildRelationship(generationSpan!, agentSpan); + + // Validate status + expect(agentSpan.status).toBeDefined(); + expect(agentSpan.status.code).toBe(1); + + console.log("โœ… Agent span validation passed"); + } + + console.log("โœ… All span structure validation passed"); + + // Verify the response + expect(result.finalOutput).toBeDefined(); + console.log("โœ… Agent response received"); + } catch (error) { + console.error("Test error:", error); + throw error; + } + }); + + it("Validate execution spans", async () => { + const azureConfig = getAzureOpenAIConfig(); + + if (!azureConfig) { + throw new Error("Azure OpenAI configuration is required"); + } + + try { + const azureClient = new AzureOpenAI({ + endpoint: azureConfig.endpoint, + deployment: azureConfig.deployment, + apiKey: azureConfig.apiKey, + apiVersion: azureConfig.apiVersion, + }); + + const addTool: any = tool({ + name: "add_numbers", + description: "Add two numbers together", + parameters: { + type: "object", + properties: { + a: { + type: "number", + description: "The first number" + }, + b: { + type: "number", + description: "The second number" + } + }, + required: ["a", "b"], + additionalProperties: false + } as any, + execute: async ({ a, b }: { a: number; b: number }) => { + const result = a + b; + return `The sum of ${a} and ${b} is ${result}`; + }, + }); + + const agentName = "Math Agent"; + const agent = new Agent({ + name: "Math Agent", + model: new OpenAIChatCompletionsModel( + azureClient as any, + process.env.AZURE_OPENAI_DEPLOYMENT!, + ), + instructions: + "You are a helpful math assistant. When asked to add numbers, use the 'add_numbers' tool.", + tools: [addTool], + }); + const prompt = "What is 15 plus 27?"; + const result = await run(agent, prompt); + + const startTime = Date.now(); + const timeout = 5000; + while (spans.length < 3 && Date.now() - startTime < timeout) { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + + // Verify we captured spans + expect(spans.length).toBeGreaterThanOrEqual(3); + console.log("Total spans captured:", spans.length); + + // Output all the spans + spans.forEach((span, idx) => { + console.log(`\n--- Span ${idx + 1} of ${spans.length} ---`); + console.log(JSON.stringify(span, null, 2)); + }); + + // Find and validate the agent span + const agentSpan = spans.find( + (span) => span.name === `invoke_agent ${agentName}`, + ); + expect(agentSpan).toBeDefined(); + + if (agentSpan) { + validateInstrumentationScope(agentSpan); + expect( + agentSpan.attributes[ + OpenTelemetryConstants.GEN_AI_OPERATION_NAME_KEY + ], + ).toBe("invoke_agent"); + expect( + agentSpan.attributes[OpenTelemetryConstants.GEN_AI_SYSTEM_KEY], + ).toBe("openai"); + console.log("โœ… Agent span validated"); + } + + // Find and validate the generation span + const generationSpan = spans.find((span) => span.name === "generation"); + expect(generationSpan).toBeDefined(); + + // Find and validate the tool execution span + const toolSpan = spans.find( + (span) => span.name === "execute_tool add_numbers", + ); + expect(toolSpan).toBeDefined(); + console.log("Validate tool execution span"); + + if (toolSpan) { + validateInstrumentationScope(toolSpan); + validateSpanProperties(toolSpan); + + // Validate tool-specific attributes + expect( + toolSpan.attributes[OpenTelemetryConstants.GEN_AI_OPERATION_NAME_KEY], + ).toBe("execute_tool"); + expect( + toolSpan.attributes[OpenTelemetryConstants.GEN_AI_SYSTEM_KEY], + ).toBe("openai"); + expect( + toolSpan.attributes[OpenTelemetryConstants.GEN_AI_TOOL_NAME_KEY], + ).toBe("add_numbers"); + expect( + toolSpan.attributes[OpenTelemetryConstants.GEN_AI_TOOL_TYPE_KEY], + ).toBe("function"); + expect( + toolSpan.attributes[OpenTelemetryConstants.GEN_AI_TOOL_ARGS_KEY], + ).toBe('{"a":15,"b":27}'); + expect( + toolSpan.attributes[OpenTelemetryConstants.GEN_AI_EVENT_CONTENT], + ).toBe("The sum of 15 and 27 is 42"); + + validateParentChildRelationship(toolSpan, agentSpan!); + + // Validate status + expect(toolSpan.status).toBeDefined(); + expect(toolSpan.status.code).toBe(1); + + console.log("โœ… Tool execution span validated"); + } + + // Verify the response + expect(result.finalOutput).toBeDefined(); + console.log("โœ… Agent response received"); + } catch (error) { + console.error("Test error:", error); + throw error; + } + }); + + /** + * Validate instrumentation scope for a span + */ + function validateInstrumentationScope(span: ReadableSpan): void { + expect(span.instrumentationScope).toBeDefined(); + expect(span.instrumentationScope.name).toBe(TEST_INSTRUMENTATION_NAME); + expect(span.instrumentationScope.version).toBe( + TEST_INSTRUMENTATION_VERSION, + ); + } + + /** + * Validate basic span properties (traceId, id, timestamp) + */ + function validateSpanProperties(span: ReadableSpan): void { + expect((span as any).traceId).toBeDefined(); + expect((span as any).id).toBeDefined(); + expect((span as any).timestamp).toBeDefined(); + } + + /** + * Validate parent-child span relationship + */ + function validateParentChildRelationship( + childSpan: ReadableSpan, + parentSpan: ReadableSpan, + ): void { + expect( + childSpan.attributes[OpenTelemetryConstants.CUSTOM_PARENT_SPAN_ID_KEY], + ).toBe(`0x${(parentSpan as any).id}`); + } +}); diff --git a/tests/observability/integration/setup.ts b/tests/observability/integration/setup.ts new file mode 100644 index 00000000..59bb2fb1 --- /dev/null +++ b/tests/observability/integration/setup.ts @@ -0,0 +1,37 @@ +// ------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------------------------ + +/** + * Global setup for all integration tests + * This file is loaded by Jest before any tests run + */ + +import { beforeAll, afterAll, afterEach } from '@jest/globals'; + +// Global setup before all tests +beforeAll(() => { + console.log('๐Ÿš€ Starting integration tests...'); + // Set up required environment variables for testing + process.env.AZURE_EXPERIMENTAL_ENABLE_ACTIVITY_SOURCE = 'true'; + process.env.AZURE_TRACING_GEN_AI_CONTENT_RECORDING_ENABLED = 'true'; + process.env.OPENAI_AGENTS_DISABLE_TRACING = 'false'; + process.env.OTEL_SDK_DISABLED = 'false'; + + // Initialize global observability/telemetry before tests + // Setup global test data and fixtures + // Set global timeout if needed + jest.setTimeout(60000); +}); + +afterAll(async () => { + console.log('๐Ÿ Integration tests completed'); + + // Clean up global resources after tests + // Export global telemetry for validation +}); + +afterEach(async () => { + // Clean up after each test (common cleanup) + // Reset global mocks, clear timers, etc. +}); diff --git a/tests/package.json b/tests/package.json index 3d2acdcd..edcb5527 100644 --- a/tests/package.json +++ b/tests/package.json @@ -9,8 +9,10 @@ "scripts": { "clean": "rimraf dist", "build": "echo 'No build needed for test package'", - "test": "jest --passWithNoTests", - "test:watch": "jest --watch", + "test": "jest --passWithNoTests --testPathIgnorePatterns=/integration/", + "test:watch": "jest --watch --testPathIgnorePatterns=/integration/", + "test:integration": "jest --config ../jest.integration.config.cjs", + "test:integration:watch": "jest --config ../jest.integration.config.cjs --watch", "ci": "npm ci", "build:all": "npm run build --workspaces" }, @@ -38,12 +40,15 @@ "@azure/monitor-opentelemetry-exporter": "*", "@modelcontextprotocol/sdk": "*", "@openai/agents": "*", + "@openai/agents-openai": "*", "@opentelemetry/api": "*", "@opentelemetry/exporter-trace-otlp-http": "*", "@opentelemetry/resources": "*", "@opentelemetry/sdk-node": "*", "@opentelemetry/sdk-trace-base": "*", - "@opentelemetry/semantic-conventions": "*" + "@opentelemetry/semantic-conventions": "*", + "dotenv": "^16.0.0", + "openai": "^4.0.0" }, "devDependencies": { "@babel/preset-typescript": "^7.27.1",