diff --git a/package.json b/package.json index 9169655..3b75845 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "build": "turbo build", "dev": "turbo dev", "lint": "turbo lint", + "validate-data": "node packages/tools/scripts/validate_data_streaming.js", "format": "prettier --write \"**/*.{ts,tsx,md}\"", "start:collector": "docker-compose -f docker/docker-compose.yaml up otel-collector -d", "stop:collector": "docker-compose -f docker/docker-compose.yaml down", diff --git a/tools/scripts/README.md b/packages/tools/scripts/README.md old mode 100644 new mode 100755 similarity index 100% rename from tools/scripts/README.md rename to packages/tools/scripts/README.md diff --git a/tools/scripts/package-lock.json b/packages/tools/scripts/package-lock.json old mode 100644 new mode 100755 similarity index 99% rename from tools/scripts/package-lock.json rename to packages/tools/scripts/package-lock.json index 860a7d1..48add03 --- a/tools/scripts/package-lock.json +++ b/packages/tools/scripts/package-lock.json @@ -50,4 +50,4 @@ } } } -} +} \ No newline at end of file diff --git a/tools/scripts/package.json b/packages/tools/scripts/package.json old mode 100644 new mode 100755 similarity index 100% rename from tools/scripts/package.json rename to packages/tools/scripts/package.json diff --git a/packages/tools/scripts/validate_data_streaming.js b/packages/tools/scripts/validate_data_streaming.js new file mode 100755 index 0000000..26428de --- /dev/null +++ b/packages/tools/scripts/validate_data_streaming.js @@ -0,0 +1,74 @@ +const { createClient } = require('@clickhouse/client'); +const dotenv = require('dotenv'); + +dotenv.config({ path: '../../../.env' }); + +const client = createClient({ + host: process.env.CLICKHOUSE_ENDPOINT, + database: process.env.CLICKHOUSE_DATABASE, + username: process.env.CLICKHOUSE_USERNAME, + password: process.env.CLICKHOUSE_PASSWORD +}); + +async function validateDataStreaming() { + console.log('Validating data streaming to ClickHouse...'); + + try { + const countQuery = await client.query({ + query: 'SELECT COUNT(*) as count FROM default.otel_traces', + format: 'JSONEachRow' + }); + const countData = await countQuery.json(); + console.log(`Found ${countData[0].count} records in ClickHouse.`); + + const latestDataQuery = await client.query({ + query: 'SELECT MAX(Timestamp) as latest FROM default.otel_traces', + format: 'JSONEachRow' + }); + const latestData = await latestDataQuery.json(); + console.log(`Latest data received at ${new Date(latestData[0].latest).toLocaleString()}`); + + const spanNamesQuery = await client.query({ + query: 'SELECT DISTINCT SpanName FROM default.otel_traces LIMIT 10', + format: 'JSONEachRow' + }); + const spanNames = await spanNamesQuery.json(); + console.log('Span names found:', spanNames.map(row => row.SpanName).join(', ')); + + const largePayloadQuery = await client.query({ + query: 'SELECT COUNT(*) as count FROM default.otel_traces WHERE length(SpanAttributes) > 10', + format: 'JSONEachRow' + }); + const largePayloadData = await largePayloadQuery.json(); + console.log(`Number of spans with large payloads: ${largePayloadData[0].count}`); + + const sdkLanguageQuery = await client.query({ + query: 'SELECT DISTINCT ResourceAttributes[\'telemetry.sdk.language\'] as sdk_language FROM default.otel_traces WHERE ResourceAttributes[\'telemetry.sdk.language\'] IS NOT NULL LIMIT 1', + format: 'JSONEachRow' + }); + const sdkLanguageData = await sdkLanguageQuery.json(); + console.log(`SDK Language: ${sdkLanguageData[0]?.sdk_language || 'Not found'}`); + + const exporterQuery = await client.query({ + query: 'SELECT DISTINCT ResourceAttributes[\'telemetry.sdk.name\'] as sdk_name FROM default.otel_traces WHERE ResourceAttributes[\'telemetry.sdk.name\'] IS NOT NULL LIMIT 1', + format: 'JSONEachRow' + }); + const exporterData = await exporterQuery.json(); + console.log(`Exporter SDK: ${exporterData[0]?.sdk_name || 'Not found'}`); + + const avgDurationQuery = await client.query({ + query: 'SELECT AVG(Duration) as avg_duration FROM default.otel_traces', + format: 'JSONEachRow' + }); + const avgDurationData = await avgDurationQuery.json(); + console.log(`Average span duration: ${avgDurationData[0].avg_duration} ns`); + + console.log('Data streaming validation completed successfully.'); + } catch (error) { + console.error('Error during data streaming validation:', error); + } finally { + await client.close(); + } +} + +validateDataStreaming(); \ No newline at end of file diff --git a/tools/scripts/validate_data_streaming.js b/tools/scripts/validate_data_streaming.js deleted file mode 100644 index 97ffe92..0000000 --- a/tools/scripts/validate_data_streaming.js +++ /dev/null @@ -1,54 +0,0 @@ -const { createClient } = require('@clickhouse/client'); -const dotenv = require('dotenv'); - -dotenv.config({ path: '../../.env' }); - -const client = createClient({ - host: process.env.CLICKHOUSE_ENDPOINT, - database: process.env.CLICKHOUSE_DATABASE, - username: process.env.CLICKHOUSE_USERNAME, - password: process.env.CLICKHOUSE_PASSWORD -}); - -async function validateDataStreaming() { - console.log('Validating data streaming to ClickHouse...'); - - try { - const dataExistsQuery = await client.query({ - query: 'SELECT COUNT(*) as count FROM otel_traces', - format: 'JSONEachRow' - }); - const dataExists = await dataExistsQuery.json(); - const count = dataExists[0].count; - - if (count === 0) { - console.error('No data found in ClickHouse. Make sure your application is sending data.'); - return; - } - - console.log(`Found ${count} records in ClickHouse.`); - - const recentDataQuery = await client.query({ - query: 'SELECT MAX(Timestamp) as latest FROM otel_traces', - format: 'JSONEachRow' - }); - const recentData = await recentDataQuery.json(); - const latestTimestamp = new Date(recentData[0].latest); - console.log(`Latest data received at ${latestTimestamp}`); - - const spanNamesQuery = await client.query({ - query: 'SELECT DISTINCT SpanName FROM otel_traces LIMIT 10', - format: 'JSONEachRow' - }); - const spanNames = await spanNamesQuery.json(); - console.log('Span names found:', spanNames.map(row => row.SpanName).join(', ')); - - console.log('Data streaming validation completed successfully.'); - } catch (error) { - console.error('Error during data streaming validation:', error); - } finally { - await client.close(); - } -} - -validateDataStreaming(); \ No newline at end of file