From 826eedd2bc8e7df8073250e01f3f833c81e51f0d Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Thu, 1 Jan 2026 21:39:29 -0500 Subject: [PATCH 01/15] chore: add performance test results to gitignore --- applications/chatops/slack-bot/.gitignore | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/applications/chatops/slack-bot/.gitignore b/applications/chatops/slack-bot/.gitignore index e372cf2..7cdc743 100644 --- a/applications/chatops/slack-bot/.gitignore +++ b/applications/chatops/slack-bot/.gitignore @@ -30,3 +30,10 @@ npm-debug.log* # TypeScript *.tsbuildinfo + +# Performance test results +performance-tests/results/*.json +performance-tests/results/*.html + +# Environment files +performance-tests/.env From 93937e5e0bbbbd3964d20d636cb9bee55e6c62b5 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Thu, 1 Jan 2026 22:54:03 -0500 Subject: [PATCH 02/15] Add Artillery performance testing infrastructure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add performance-tests directory with Artillery configurations - Add CloudWatch Logs analysis scripts (analyze-performance.sh, analyze-e2e-json.sh) - Add Slack signature processor for load testing - Add test scenarios: echo-only, echo-light, full config πŸ€– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- applications/chatops/slack-bot/Makefile | 130 +++ .../slack-bot/performance-tests/.env.example | 12 + .../slack-bot/performance-tests/README.md | 371 +++++++ .../performance-tests/analyze-e2e-json.sh | 144 +++ .../performance-tests/analyze-performance.sh | 429 ++++++++ .../performance-tests/artillery-config.yml | 88 ++ .../artillery-echo-light.yml | 60 ++ .../performance-tests/artillery-echo-only.yml | 54 + .../performance-tests/render-report.js | 978 ++++++++++++++++++ .../slack-signature-processor.js | 132 +++ .../slack-bot/performance-tests/test-curl.sh | 93 ++ .../performance-tests/test-signature.js | 47 + .../performance-tests/view-results.sh | 140 +++ 13 files changed, 2678 insertions(+) create mode 100644 applications/chatops/slack-bot/performance-tests/.env.example create mode 100644 applications/chatops/slack-bot/performance-tests/README.md create mode 100755 applications/chatops/slack-bot/performance-tests/analyze-e2e-json.sh create mode 100755 applications/chatops/slack-bot/performance-tests/analyze-performance.sh create mode 100644 applications/chatops/slack-bot/performance-tests/artillery-config.yml create mode 100644 applications/chatops/slack-bot/performance-tests/artillery-echo-light.yml create mode 100644 applications/chatops/slack-bot/performance-tests/artillery-echo-only.yml create mode 100644 applications/chatops/slack-bot/performance-tests/render-report.js create mode 100644 applications/chatops/slack-bot/performance-tests/slack-signature-processor.js create mode 100755 applications/chatops/slack-bot/performance-tests/test-curl.sh create mode 100644 applications/chatops/slack-bot/performance-tests/test-signature.js create mode 100755 applications/chatops/slack-bot/performance-tests/view-results.sh diff --git a/applications/chatops/slack-bot/Makefile b/applications/chatops/slack-bot/Makefile index 25f346c..01ed22a 100644 --- a/applications/chatops/slack-bot/Makefile +++ b/applications/chatops/slack-bot/Makefile @@ -48,6 +48,18 @@ help: @echo " make deploy- - Upload to S3 β†’ Deploy" @echo " make deploy-build - Deploy build worker from local (default)" @echo "" + @echo "$(GREEN)Performance Testing:$(NC)" + @echo " make perf-test - Full test (~12 min, all commands)" + @echo " make perf-test-echo - Echo-only test (~7 min)" + @echo " make perf-test-quick - Quick test (2 min)" + @echo " make perf-test-results - View latest test results (terminal)" + @echo " make perf-test-analyze - Analyze CloudWatch metrics (last 15 min)" + @echo " make perf-test-analyze-test - Analyze from latest Artillery test" + @echo " make perf-test-analyze-test-quiet - Analyze from latest test (auto mode)" + @echo " make perf-test-json - Export metrics to JSON (dashboard)" + @echo " make perf-test-summary - Quick summary of latest test" + @echo " make perf-test-clean - Clean test results" + @echo "" @echo "$(GREEN)Deploy Lambda (Advanced):$(NC)" @echo " make deploy--local - Deploy from local dist/ (fast, no upload)" @echo "" @@ -224,3 +236,121 @@ clean-all: clean @echo "$(BLUE)Cleaning all dependencies...$(NC)" rm -rf node_modules @echo "$(GREEN)βœ“ Deep clean complete$(NC)" + +# ----------------------------------------------------------------------------- +# Performance Testing +# ----------------------------------------------------------------------------- + +perf-test: perf-test-install + @echo "$(BLUE)Running Artillery performance test...$(NC)" + @echo "$(YELLOW)Environment: $(ENVIRONMENT)$(NC)" + @echo "$(YELLOW)Region: $(REGION)$(NC)" + @echo "" + @if [ -z "$(API_GATEWAY_URL)" ]; then \ + echo "$(YELLOW)⚠ API_GATEWAY_URL not set. Getting from Terragrunt...$(NC)"; \ + export API_GATEWAY_URL=$$(cd $(SANDBOX_ROOT)/slack-api-gateway && terragrunt output -raw api_gateway_url 2>/dev/null || echo ""); \ + if [ -z "$$API_GATEWAY_URL" ]; then \ + echo "$(YELLOW)βœ— Could not get API Gateway URL from Terragrunt$(NC)"; \ + echo "$(YELLOW) Please set manually: make perf-test API_GATEWAY_URL=https://xxx.execute-api.ca-central-1.amazonaws.com$(NC)"; \ + exit 1; \ + fi; \ + echo "$(GREEN)βœ“ API Gateway URL: $$API_GATEWAY_URL$(NC)"; \ + export ENVIRONMENT=$(ENVIRONMENT) AWS_REGION=$(REGION); \ + cd performance-tests && artillery run artillery-config.yml --output results/test-$$(date +%Y%m%d-%H%M%S).json; \ + else \ + export ENVIRONMENT=$(ENVIRONMENT) AWS_REGION=$(REGION); \ + cd performance-tests && artillery run artillery-config.yml --output results/test-$$(date +%Y%m%d-%H%M%S).json; \ + fi + +perf-test-echo: perf-test-install + @echo "$(BLUE)Running Echo-only performance test (7 minutes, max 40 req/s)...$(NC)" + @echo "$(YELLOW)Environment: $(ENVIRONMENT)$(NC)" + @echo "$(YELLOW)Region: $(REGION)$(NC)" + @echo "$(YELLOW)⚠ Max load: 40 req/s (API Gateway limit: 50 req/s)$(NC)" + @echo "" + @if [ -z "$(API_GATEWAY_URL)" ]; then \ + echo "$(YELLOW)⚠ API_GATEWAY_URL not set. Getting from Terragrunt...$(NC)"; \ + export API_GATEWAY_URL=$$(cd $(SANDBOX_ROOT)/slack-api-gateway && terragrunt output -raw api_gateway_url 2>/dev/null || echo ""); \ + if [ -z "$$API_GATEWAY_URL" ]; then \ + echo "$(YELLOW)βœ— Could not get API Gateway URL from Terragrunt$(NC)"; \ + echo "$(YELLOW) Please set manually: make perf-test-echo API_GATEWAY_URL=https://xxx.execute-api.ca-central-1.amazonaws.com$(NC)"; \ + exit 1; \ + fi; \ + echo "$(GREEN)βœ“ API Gateway URL: $$API_GATEWAY_URL$(NC)"; \ + export ENVIRONMENT=$(ENVIRONMENT) AWS_REGION=$(REGION); \ + cd performance-tests && artillery run artillery-echo-light.yml --output results/echo-test-$$(date +%Y%m%d-%H%M%S).json; \ + else \ + export ENVIRONMENT=$(ENVIRONMENT) AWS_REGION=$(REGION); \ + cd performance-tests && artillery run artillery-echo-light.yml --output results/echo-test-$$(date +%Y%m%d-%H%M%S).json; \ + fi + +perf-test-quick: perf-test-install + @echo "$(BLUE)Running quick Artillery test (2 minutes)...$(NC)" + @export ENVIRONMENT=$(ENVIRONMENT) AWS_REGION=$(REGION); \ + if [ -z "$(API_GATEWAY_URL)" ]; then \ + export API_GATEWAY_URL=$$(cd $(SANDBOX_ROOT)/slack-api-gateway && terragrunt output -raw api_gateway_url 2>/dev/null || echo "https://placeholder.execute-api.ca-central-1.amazonaws.com"); \ + fi; \ + cd performance-tests && artillery quick --count 100 --num 10 $$API_GATEWAY_URL/slack + +perf-test-results: + @echo "$(BLUE)Viewing latest test results...$(NC)" + @cd performance-tests && ./view-results.sh + +perf-test-report: + @echo "$(BLUE)Generating HTML report from latest test...$(NC)" + @REPORT_JSON=$${REPORT_JSON:-$$(ls -t performance-tests/results/*-test-*.json 2>/dev/null | grep -v '\.metrics\.json' | head -n1)}; \ + if [ -z "$$REPORT_JSON" ]; then \ + echo "$(YELLOW)βœ— No test results found$(NC)"; \ + exit 1; \ + fi; \ + REPORT_HTML=$${REPORT_HTML:-$${REPORT_JSON%.json}.html}; \ + node performance-tests/render-report.js $$REPORT_JSON $$REPORT_HTML; \ + echo "$(GREEN)βœ“ Report written: $$REPORT_HTML$(NC)" + +perf-test-summary: + @echo "$(BLUE)Quick summary of latest test...$(NC)" + @LATEST=$$(ls -t performance-tests/results/*-test-*.json 2>/dev/null | head -n1); \ + if [ -z "$$LATEST" ]; then \ + echo "$(YELLOW)βœ— No test results found$(NC)"; \ + exit 1; \ + fi; \ + echo "$(GREEN)File: $$LATEST$(NC)"; \ + echo ""; \ + jq -r '"Requests: " + (.aggregate.counters["http.requests"] | tostring), \ + "P50: " + (.aggregate.summaries["http.response_time"].median | tostring) + "ms", \ + "P95: " + (.aggregate.summaries["http.response_time"].p95 | tostring) + "ms", \ + "P99: " + (.aggregate.summaries["http.response_time"].p99 | tostring) + "ms", \ + "Errors: " + ((.aggregate.counters["errors.total"] // 0) | tostring)' \ + $$LATEST 2>/dev/null || cat $$LATEST | grep -A 20 "aggregate" + +perf-test-install: + @echo "$(BLUE)Checking Artillery installation...$(NC)" + @if ! command -v artillery > /dev/null; then \ + echo "$(YELLOW)Installing Artillery globally...$(NC)"; \ + npm install -g artillery artillery-plugin-metrics-by-endpoint; \ + fi + @echo "$(BLUE)Installing test dependencies...$(NC)" + @cd performance-tests && npm install --no-save @aws-sdk/client-ssm 2>/dev/null || true + @echo "$(GREEN)βœ“ Artillery ready$(NC)" + +perf-test-clean: + @echo "$(BLUE)Cleaning performance test results...$(NC)" + rm -rf performance-tests/results/*.json + rm -rf performance-tests/results/*.html + @echo "$(GREEN)βœ“ Performance test results cleaned$(NC)" + +perf-test-analyze: + @echo "$(BLUE)Analyzing performance test results from CloudWatch...$(NC)" + @cd performance-tests && ./analyze-performance.sh 15 +perf-test-analyze-test: + @echo "$(BLUE)Analyzing from latest Artillery test result...$(NC)" + @cd performance-tests && ./analyze-performance.sh --from-test + +perf-test-analyze-test-quiet: + @echo "$(BLUE)Analyzing from latest Artillery test result (quiet mode)...$(NC)" + @cd performance-tests && ./analyze-performance.sh --from-test --quiet 2>/dev/null + +perf-test-json: + @echo "$(BLUE)Exporting E2E metrics to JSON...$(NC)" + @cd performance-tests && ./analyze-e2e-json.sh >/dev/null + @echo "$(GREEN)βœ“ Metrics file created in performance-tests/results/$(NC)" \ No newline at end of file diff --git a/applications/chatops/slack-bot/performance-tests/.env.example b/applications/chatops/slack-bot/performance-tests/.env.example new file mode 100644 index 0000000..e9b2b6c --- /dev/null +++ b/applications/chatops/slack-bot/performance-tests/.env.example @@ -0,0 +1,12 @@ +# Performance Testing Environment Variables +# Copy this file to .env and fill in your values (DO NOT commit .env) + +# Environment to test (plt, dev, prd) +ENVIRONMENT=plt + +# AWS Region +AWS_REGION=ca-central-1 + +# API Gateway URL with full path (get from Terragrunt output or AWS Console) +# Example: https://xxxxxx.execute-api.ca-central-1.amazonaws.com/prod/slack +API_GATEWAY_URL= diff --git a/applications/chatops/slack-bot/performance-tests/README.md b/applications/chatops/slack-bot/performance-tests/README.md new file mode 100644 index 0000000..50961b8 --- /dev/null +++ b/applications/chatops/slack-bot/performance-tests/README.md @@ -0,0 +1,371 @@ +# Slack Bot Performance Testing + +End-to-end performance testing for the Slack bot architecture using Artillery. + +## Quick Start + +```bash +# Run full performance test +make perf-test + +# Run quick 2-minute test +make perf-test-quick + +# Generate HTML report from latest test +make perf-test-report +``` + +## Prerequisites + +1. **AWS Credentials** - Configured with access to: + - SSM Parameter Store (to fetch Slack signing secret) + - API Gateway endpoint + +2. **Slack Signing Secret** - Must be stored in SSM: + ``` + /laco/plt/aws/secrets/slack/signing-secret + ``` + +3. **API Gateway URL** - Full URL including path, auto-detected from Terragrunt, or set manually: + ```bash + make perf-test API_GATEWAY_URL=https://xxx.execute-api.ca-central-1.amazonaws.com/prod/slack + ``` + +## Test Scenarios + +The Artillery test runs through 5 phases: + +1. **Warm-up** (60s @ 2 req/s) + - Ensures Lambda functions are warm + - Prevents cold start bias + +2. **Ramp-up** (180s, 5 β†’ 20 req/s) + - Gradually increases load + - Tests auto-scaling behavior + +3. **Sustained Load** (300s @ 20 req/s) + - Steady-state performance + - Establishes baseline metrics + +4. **High Load** (120s, 20 β†’ 45 req/s) + - Approaches API Gateway throttle limit (50 req/s) + - Tests behavior under stress + +5. **Cool-down** (60s @ 5 req/s) + - Allows system to stabilize + +**Total Duration:** ~12 minutes + +## Command Distribution + +Matches realistic usage patterns: + +- `/echo` (40%) - Fast command, minimal processing +- `/status` (30%) - Medium complexity, service checks +- `/deploy` (20%) - Slower, deployment simulation +- `/build` (10%) - Slower, GitHub API integration + +## Performance Thresholds + +Tests fail if any threshold is exceeded: + +- **Error Rate:** < 1% +- **P95 Latency:** < 3000ms (3 seconds) +- **P99 Latency:** < 5000ms (5 seconds) + +## Output + +### Console Output + +Real-time statistics during the test: +``` +Scenarios launched: 1234 +Scenarios completed: 1200 +Requests completed: 1200 +Mean response/sec: 19.2 +Response time (msec): + min: 245 + max: 4521 + median: 892 + p95: 2134 + p99: 3456 +``` + +### JSON Results + +Saved to `performance-tests/results/test-YYYYMMDD-HHMMSS.json` + +Contains: +- Request/response timings +- Error counts and types +- Percentile distributions +- Phase-by-phase breakdown + +### HTML Report + +Generate with: `make perf-test-report` + +Includes: +- Interactive charts +- Timeline view +- Request distribution graphs +- Error analysis + +## Interpreting Results + +### Key Metrics to Watch + +1. **P95/P99 Latency** + - Target: < 3000ms / < 5000ms + - High values indicate bottlenecks + +2. **Error Rate** + - Target: < 1% + - Errors by status code: + - 429: API Gateway throttling + - 500: Lambda errors + - 503: Service unavailable + +3. **Response Time Distribution** + - Should follow command complexity: + - Echo: 500-1000ms + - Status: 1000-2000ms + - Deploy/Build: 2000-3000ms + +### Common Issues + +#### High P95/P99 Latency + +**Symptoms:** +- P99 > 5000ms +- Wide gap between median and P99 + +**Possible Causes:** +- Cold starts +- Worker concurrency limits (5 concurrent) +- Queue backlog + +**Investigation:** +1. Check CloudWatch Lambda concurrent executions +2. Check SQS queue depth during test +3. Review X-Ray traces for slow subsegments + +#### API Gateway Throttling (429 errors) + +**Symptoms:** +- 429 status codes +- Errors during high-load phase + +**Root Cause:** +- Default 50 req/s throttle limit exceeded + +**Solutions:** +- Request quota increase +- Reduce test load +- Implement request queuing + +#### Lambda Errors (500/502/503) + +**Symptoms:** +- 5xx status codes +- Errors in specific commands + +**Investigation:** +1. Check CloudWatch Logs for Lambda errors +2. Review error messages in Artillery output +3. Check Lambda timeout configuration + +## Advanced Usage + +### Custom Test Duration + +Edit `artillery-config.yml` phases: + +```yaml +phases: + - duration: 120 # 2 minutes instead of default + arrivalRate: 10 +``` + +### Test Single Command + +Create custom scenario file: + +```yaml +# artillery-echo-only.yml +config: + target: "{{ $processEnvironment.API_GATEWAY_URL }}" + phases: + - duration: 60 + arrivalRate: 20 + processor: "./slack-signature-processor.js" + +scenarios: + - name: "Echo Only" + flow: + - post: + url: "/slack" + beforeRequest: "generateSlackSignature" + body: "token=test&command=/echo&text=test&..." +``` + +Run: `artillery run artillery-echo-only.yml` + +### Different Environment + +```bash +# Test against dev environment +make perf-test ENVIRONMENT=dev + +# Test against production (careful!) +make perf-test ENVIRONMENT=prd +``` + +### Manual Artillery Run + +```bash +# Set environment variables +export API_GATEWAY_URL="https://xxx.execute-api.ca-central-1.amazonaws.com" +export ENVIRONMENT="plt" +export AWS_REGION="ca-central-1" + +# Run test +cd performance-tests +artillery run artillery-config.yml --output results/test.json + +# Generate HTML report +node render-report.js results/test.json results/report.html +# +# Or from repo root: +# make perf-test-report REPORT_JSON=performance-tests/results/test.json +``` +Note: The report uses Chart.js from a CDN, so charts require network access when viewing. + +## Monitoring During Tests + +### CloudWatch Dashboard + +Monitor in real-time: +``` +https://console.aws.amazon.com/cloudwatch/home?region=ca-central-1#dashboards:name=SlackBot-Performance-PLT +``` + +### CloudWatch Logs Insights + +Query during test: + +```sql +fields @timestamp, correlationId, e2eLatency, component +| filter component = "echo-worker" +| stats avg(e2eLatency) as avg, percentile(e2eLatency, 95) as p95 by bin(1m) +``` + +### X-Ray Service Map + +View distributed trace: +``` +https://console.aws.amazon.com/xray/home?region=ca-central-1#/service-map +``` + +## Baseline Performance + +Establish baseline before architectural changes: + +```bash +# Run full test +make perf-test + +# Generate report +make perf-test-report + +# Archive results +cp performance-tests/results/test-*.json baseline-YYYYMMDD.json +``` + +Compare before/after: +1. P50/P95/P99 latencies +2. Error rates +3. Throughput (req/s) +4. Resource utilization + +## Troubleshooting + +### "Failed to fetch Slack signing secret from SSM" + +**Solution:** +```bash +# Verify secret exists +aws ssm get-parameter \ + --name /laco/plt/aws/secrets/slack/signing-secret \ + --with-decryption \ + --region ca-central-1 + +# Check AWS credentials +aws sts get-caller-identity +``` + +### "Could not get API Gateway URL from Terragrunt" + +**Solution:** +```bash +# Get URL manually +cd ../../../../cloud-sandbox/aws/10-plt/slack-api-gateway +terragrunt output api_gateway_url + +# Set manually +make perf-test API_GATEWAY_URL=https://xxx.execute-api.ca-central-1.amazonaws.com +``` + +### "Artillery command not found" + +**Solution:** +```bash +# Install globally +npm install -g artillery + +# Or use npx +npx artillery run artillery-config.yml +``` + +### High error rate during test + +**Check:** +1. Is the PLT environment healthy? +2. Are Lambdas deployed? +3. Is Slack signing secret correct? +4. Are EventBridge rules configured? +5. Are SQS queues created? + +```bash +# Verify stack health +cd ../../../../cloud-sandbox/aws/10-plt +terragrunt run-all output +``` + +## Files + +``` +performance-tests/ +β”œβ”€β”€ README.md # This file +β”œβ”€β”€ artillery-config.yml # Main Artillery configuration +β”œβ”€β”€ slack-signature-processor.js # Slack signature generation +└── results/ # Test results (gitignored) + β”œβ”€β”€ test-YYYYMMDD-HHMMSS.json # Raw test data + └── report-YYYYMMDD-HHMMSS.html # HTML report +``` + +## Next Steps + +After establishing baseline with Artillery: + +1. **Component Testing** - Test individual components with AWS SDK +2. **Identify Bottlenecks** - Compare Artillery E2E vs component latencies +3. **Optimize** - Focus on slowest components +4. **Re-test** - Validate improvements with Artillery +5. **Production Monitoring** - Set up CloudWatch alarms based on baselines + +## References + +- [Artillery Documentation](https://www.artillery.io/docs) +- [Performance Testing Guide](/cloud-control-plane/docs/guides/performance-testing-guide.md) +- [Slack Bot Architecture](/cloud-control-plane/docs/architecture/) diff --git a/applications/chatops/slack-bot/performance-tests/analyze-e2e-json.sh b/applications/chatops/slack-bot/performance-tests/analyze-e2e-json.sh new file mode 100755 index 0000000..51539cc --- /dev/null +++ b/applications/chatops/slack-bot/performance-tests/analyze-e2e-json.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# Extract E2E performance metrics from Artillery test and CloudWatch Logs +# Output: JSON format for dashboard integration + +set -e + +ENVIRONMENT="${ENVIRONMENT:-plt}" +REGION="ca-central-1" + +# Find latest Artillery test result (exclude .metrics.json files) +LATEST_RESULT=$(ls -t results/*.json 2>/dev/null | grep -v '\.metrics\.json' | head -1) + +if [ -z "$LATEST_RESULT" ]; then + echo '{"error": "No Artillery test results found"}' >&2 + exit 1 +fi + +echo "Analyzing: $LATEST_RESULT" >&2 + +# Extract timestamps from Artillery JSON +read START_MS END_MS < <( + node -pe " + const data = require('./$LATEST_RESULT'); + const agg = data.aggregate || data.rawAggregate || {}; + const start = agg.firstMetricAt || 0; + const end = agg.lastMetricAt || 0; + \`\${start} \${end}\` + " +) + +if [ "$START_MS" = "0" ] || [ "$END_MS" = "0" ]; then + echo '{"error": "Could not extract timestamps from Artillery result"}' >&2 + exit 1 +fi + +# Helper function to execute query and wait for results +query_logs() { + local log_group=$1 + local query_string=$2 + local metric_name=$3 + + # Convert milliseconds to seconds for AWS Logs API + local start_sec=$((START_MS / 1000)) + local end_sec=$((END_MS / 1000)) + + local query_id=$(aws logs start-query \ + --log-group-name "$log_group" \ + --start-time $start_sec \ + --end-time $end_sec \ + --region $REGION \ + --query-string "$query_string" \ + --query 'queryId' \ + --output text) + + # Wait for query to complete + sleep 5 + + aws logs get-query-results \ + --query-id "$query_id" \ + --region $REGION \ + --output json | jq -r '.results' +} + +# 1. Router Lambda Performance +echo "Querying Router Lambda metrics..." >&2 +ROUTER_METRICS=$(query_logs \ + "/aws/lambda/laco-${ENVIRONMENT}-slack-router" \ + "fields @duration | filter @type = \"REPORT\" | stats count() as invocations, avg(@duration) as avg_ms, percentile(@duration, 50) as p50_ms, percentile(@duration, 95) as p95_ms, percentile(@duration, 99) as p99_ms, max(@duration) as max_ms" \ + "router") + +# 2. Worker Lambda Performance +echo "Querying Worker Lambda metrics..." >&2 +WORKER_METRICS=$(query_logs \ + "/aws/lambda/laco-${ENVIRONMENT}-chatbot-echo-worker" \ + "fields @duration | filter @type = \"REPORT\" | stats count() as invocations, avg(@duration) as avg_ms, percentile(@duration, 50) as p50_ms, percentile(@duration, 95) as p95_ms, percentile(@duration, 99) as p99_ms, max(@duration) as max_ms" \ + "worker") + +# 3. End-to-End Latency (if available) +echo "Querying E2E latency..." >&2 +E2E_METRICS=$(query_logs \ + "/aws/lambda/laco-${ENVIRONMENT}-chatbot-echo-worker" \ + "fields totalE2eMs, queueWaitMs, workerDurationMs | filter totalE2eMs > 0 | stats count() as requests, avg(totalE2eMs) as avg_e2e_ms, percentile(totalE2eMs, 95) as p95_e2e_ms, percentile(totalE2eMs, 99) as p99_e2e_ms, avg(queueWaitMs) as avg_queue_ms" \ + "e2e" 2>/dev/null || echo "[]") + +# 4. Error Analysis +echo "Querying errors..." >&2 +ROUTER_ERRORS=$(query_logs \ + "/aws/lambda/laco-${ENVIRONMENT}-slack-router" \ + "fields @message | filter level = \"error\" or @message like /ERROR/ | stats count() as error_count" \ + "router_errors" 2>/dev/null || echo "[]") + +WORKER_ERRORS=$(query_logs \ + "/aws/lambda/laco-${ENVIRONMENT}-chatbot-echo-worker" \ + "fields @message | filter level = \"error\" or @message like /ERROR/ | stats count() as error_count" \ + "worker_errors" 2>/dev/null || echo "[]") + +# Parse Artillery metrics +ARTILLERY_SUMMARY=$(node -pe " + const data = require('./$LATEST_RESULT'); + const summary = data.summary || {}; + JSON.stringify({ + requests: summary.requests || 0, + responses: summary.responses || 0, + errors: summary.errors || 0, + errorRate: summary.errorRate || 0, + avgRps: summary.avgRps || 0, + p50: summary.p50 || summary.median || 0, + p95: summary.p95 || 0, + p99: summary.p99 || 0, + durationMs: summary.durationMs || 0 + }); +") + +# Determine output file name (testname.metrics.json) +TESTNAME=$(basename "$LATEST_RESULT" .json) +METRICS_FILE="results/${TESTNAME}.metrics.json" + +# Build final JSON output +cat < "$METRICS_FILE" +{ + "timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)", + "environment": "$ENVIRONMENT", + "testFile": "$LATEST_RESULT", + "timeRange": { + "startMs": $START_MS, + "endMs": $END_MS, + "durationMs": $((END_MS - START_MS)) + }, + "artillery": $ARTILLERY_SUMMARY, + "cloudwatch": { + "router": $(echo "$ROUTER_METRICS" | jq -r 'if type == "array" and length > 0 then .[0] | map({(.field): .value}) | add else {} end'), + "worker": $(echo "$WORKER_METRICS" | jq -r 'if type == "array" and length > 0 then .[0] | map({(.field): .value}) | add else {} end'), + "e2e": $(echo "$E2E_METRICS" | jq -r 'if type == "array" and length > 0 then .[0] | map({(.field): .value}) | add else {} end'), + "errors": { + "router": $(echo "$ROUTER_ERRORS" | jq -r 'if type == "array" and length > 0 then (.[0] | map(select(.field == "error_count") | .value) | .[0] // 0) else 0 end'), + "worker": $(echo "$WORKER_ERRORS" | jq -r 'if type == "array" and length > 0 then (.[0] | map(select(.field == "error_count") | .value) | .[0] // 0) else 0 end') + } + } +} +EOF + +echo "" >&2 +echo "βœ“ Metrics saved to: $METRICS_FILE" >&2 +echo "Analysis complete!" >&2 diff --git a/applications/chatops/slack-bot/performance-tests/analyze-performance.sh b/applications/chatops/slack-bot/performance-tests/analyze-performance.sh new file mode 100755 index 0000000..ae3a603 --- /dev/null +++ b/applications/chatops/slack-bot/performance-tests/analyze-performance.sh @@ -0,0 +1,429 @@ +#!/bin/bash +# Analyze performance test results from CloudWatch Logs + +set -e + +ENVIRONMENT="${ENVIRONMENT:-plt}" +REGION="ca-central-1" + +# Check for flags +OUTPUT_JSON=false +USE_TEST_RESULT=false +QUIET_MODE=false + +for arg in "$@"; do + case $arg in + --from-test) + USE_TEST_RESULT=true + shift + ;; + --json) + OUTPUT_JSON=true + shift + ;; + --quiet|-q) + QUIET_MODE=true + shift + ;; + esac +done + +# Helper: echo only if not quiet +echo_info() { + if [ "$QUIET_MODE" = false ]; then + echo "$@" + fi +} + +# Helper: macOS/Linux compatible date conversion +timestamp_to_date() { + local ts=$1 + if [ "$(uname)" = "Darwin" ]; then + date -r "$ts" '+%Y-%m-%d %H:%M:%S' + else + date -d "@$ts" '+%Y-%m-%d %H:%M:%S' + fi +} + +# Helper: Calculate time N minutes ago (macOS/Linux compatible) +minutes_ago_timestamp() { + local minutes=$1 + local now=$(date +%s) + echo $((now - (minutes * 60))) +} + + +# Determine time range +if [ "$USE_TEST_RESULT" = true ]; then + # Use latest Artillery test result + LATEST_RESULT=$(ls -t results/*.json 2>/dev/null | head -1) + + if [ -z "$LATEST_RESULT" ]; then + echo "Error: No Artillery test results found in results/ directory" + exit 1 + fi + + echo "Using time range from: $LATEST_RESULT" + + # Extract timestamps from Artillery JSON + read START_TIMESTAMP END_TIMESTAMP < <( + node -pe " + const data = require('./$LATEST_RESULT'); + const start = data.aggregate.firstMetricAt || data.rawAggregate.firstMetricAt; + const end = data.aggregate.lastMetricAt || data.rawAggregate.lastMetricAt; + \`\${start} \${end}\` + " + ) + + if [ -z "$START_TIMESTAMP" ] || [ -z "$END_TIMESTAMP" ]; then + echo "Error: Could not extract timestamps from $LATEST_RESULT" + exit 1 + fi + + START_TIME_HUMAN=$(timestamp_to_date $((START_TIMESTAMP / 1000))) + END_TIME_HUMAN=$(timestamp_to_date $((END_TIMESTAMP / 1000))) + + echo_info "Test window: $START_TIME_HUMAN ~ $END_TIME_HUMAN" + echo_info "" +else + # Use traditional time range (last N minutes) + START_TIME="${1:-15}" + echo_info "Analyzing last ${START_TIME} minutes of logs..." + echo_info "" + + # Calculate timestamps (seconds, for AWS CLI) + END_TIMESTAMP=$(date +%s) + START_TIMESTAMP=$((END_TIMESTAMP - (START_TIME * 60))) +fi + +# Initialize JSON output structure +if [ "$OUTPUT_JSON" = true ]; then + JSON_OUTPUT="{\"timestamp\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\",\"environment\":\"$ENVIRONMENT\",\"timeRange\":{\"start\":$((START_TIMESTAMP * 1000)),\"end\":$((END_TIMESTAMP * 1000))},\"metrics\":{}}" +fi + +echo_info "========================================" +echo_info "Component Performance Analysis" +echo_info "========================================" +echo_info "Environment: ${ENVIRONMENT}" +echo_info "" + +# 0. End-to-End Latency (if correlation ID is present) +echo_info "0. End-to-End Latency (Router β†’ Worker)" +echo_info "----------------------------------------" +echo_info "Note: Requires correlation ID tracking in logs" +echo_info "" +aws logs start-query \ + --log-group-names "/aws/lambda/laco-${ENVIRONMENT}-slack-router" "/aws/lambda/laco-${ENVIRONMENT}-chatbot-echo-worker" \ + --start-time ${START_TIMESTAMP} \ + --end-time ${END_TIMESTAMP} \ + --region ${REGION} \ + --query-string ' +fields @timestamp, @message, @logStream +| filter @message like /correlationId/ +| parse @message "*correlationId*:*\"*\"*" as prefix, key, correlationId, suffix +| stats earliest(@timestamp) as start, latest(@timestamp) as end by correlationId +| filter isPresent(start) and isPresent(end) +| fields correlationId, (end - start) as e2e_latency_ms +| stats + count() as requests, + avg(e2e_latency_ms) as avg_e2e_ms, + percentile(e2e_latency_ms, 50) as p50_e2e_ms, + percentile(e2e_latency_ms, 95) as p95_e2e_ms, + percentile(e2e_latency_ms, 99) as p99_e2e_ms +' > /tmp/e2e-query.json 2>/dev/null || echo " ⚠ E2E tracking not available (correlation ID not found in logs)" + +if [ -f /tmp/e2e-query.json ]; then + E2E_QUERY_ID=$(cat /tmp/e2e-query.json | jq -r '.queryId' 2>/dev/null) + if [ "$E2E_QUERY_ID" != "null" ] && [ -n "$E2E_QUERY_ID" ]; then + sleep 8 + aws logs get-query-results --query-id ${E2E_QUERY_ID} --region ${REGION} --output table 2>/dev/null || echo " ⚠ Query failed" + fi +fi + +echo "" + +# 1. Router Lambda Performance +echo "1. API Gateway β†’ Router Lambda" +echo "----------------------------------------" +aws logs start-query \ + --log-group-name "/aws/lambda/laco-${ENVIRONMENT}-slack-router" \ + --start-time ${START_TIMESTAMP} \ + --end-time ${END_TIMESTAMP} \ + --region ${REGION} \ + --query-string ' +fields @timestamp, @duration, @billedDuration, @memorySize, @maxMemoryUsed +| filter @type = "REPORT" +| stats + count() as invocations, + avg(@duration) as avg_duration_ms, + percentile(@duration, 50) as p50_ms, + percentile(@duration, 95) as p95_ms, + percentile(@duration, 99) as p99_ms, + max(@duration) as max_ms, + avg(@maxMemoryUsed / 1024 / 1024) as avg_memory_mb, + max(@maxMemoryUsed / 1024 / 1024) as max_memory_mb +' > /tmp/router-query.json + +ROUTER_QUERY_ID=$(cat /tmp/router-query.json | jq -r '.queryId') + +# Wait for query to complete +sleep 5 + +aws logs get-query-results \ + --query-id ${ROUTER_QUERY_ID} \ + --region ${REGION} \ + --output table + +echo "" + +# 2. Echo Worker Lambda Performance +echo "2. Echo Worker Lambda" +echo "----------------------------------------" +aws logs start-query \ + --log-group-name "/aws/lambda/laco-${ENVIRONMENT}-chatbot-echo-worker" \ + --start-time ${START_TIMESTAMP} \ + --end-time ${END_TIMESTAMP} \ + --region ${REGION} \ + --query-string ' +fields @timestamp, @duration, @billedDuration, @memorySize, @maxMemoryUsed +| filter @type = "REPORT" +| stats + count() as invocations, + avg(@duration) as avg_duration_ms, + percentile(@duration, 50) as p50_ms, + percentile(@duration, 95) as p95_ms, + percentile(@duration, 99) as p99_ms, + max(@duration) as max_ms, + avg(@maxMemoryUsed / 1024 / 1024) as avg_memory_mb, + max(@maxMemoryUsed / 1024 / 1024) as max_memory_mb +' > /tmp/worker-query.json + +WORKER_QUERY_ID=$(cat /tmp/worker-query.json | jq -r '.queryId') + +sleep 5 + +aws logs get-query-results \ + --query-id ${WORKER_QUERY_ID} \ + --region ${REGION} \ + --output table + +echo "" + +# 3. Error Analysis +echo "3. Error Analysis" +echo "----------------------------------------" +echo "Router Errors:" +aws logs start-query \ + --log-group-name "/aws/lambda/laco-${ENVIRONMENT}-slack-router" \ + --start-time ${START_TIMESTAMP} \ + --end-time ${END_TIMESTAMP} \ + --region ${REGION} \ + --query-string ' +fields @timestamp, @message +| filter @message like /ERROR/ or @message like /Invalid signature/ +| stats count() as error_count by @message +| limit 20 +' > /tmp/router-errors.json + +ROUTER_ERROR_ID=$(cat /tmp/router-errors.json | jq -r '.queryId') +sleep 5 +aws logs get-query-results --query-id ${ROUTER_ERROR_ID} --region ${REGION} --output table + +echo "" +echo "Worker Errors:" +aws logs start-query \ + --log-group-name "/aws/lambda/laco-${ENVIRONMENT}-chatbot-echo-worker" \ + --start-time ${START_TIMESTAMP} \ + --end-time ${END_TIMESTAMP} \ + --region ${REGION} \ + --query-string ' +fields @timestamp, @message +| filter @message like /ERROR/ or level = "error" +| stats count() as error_count by @message +| limit 20 +' > /tmp/worker-errors.json + +WORKER_ERROR_ID=$(cat /tmp/worker-errors.json | jq -r '.queryId') +sleep 5 +aws logs get-query-results --query-id ${WORKER_ERROR_ID} --region ${REGION} --output table + +echo "" + +# 4. Cold Starts +echo "4. Cold Start Analysis" +echo "----------------------------------------" +echo "Router Cold Starts:" +aws logs start-query \ + --log-group-name "/aws/lambda/laco-${ENVIRONMENT}-slack-router" \ + --start-time ${START_TIMESTAMP} \ + --end-time ${END_TIMESTAMP} \ + --region ${REGION} \ + --query-string ' +fields @timestamp, @initDuration +| filter @type = "REPORT" and ispresent(@initDuration) +| stats + count() as cold_starts, + avg(@initDuration) as avg_init_ms, + max(@initDuration) as max_init_ms +' > /tmp/router-cold.json + +ROUTER_COLD_ID=$(cat /tmp/router-cold.json | jq -r '.queryId') +sleep 5 +aws logs get-query-results --query-id ${ROUTER_COLD_ID} --region ${REGION} --output table + +echo "" +echo "Worker Cold Starts:" +aws logs start-query \ + --log-group-name "/aws/lambda/laco-${ENVIRONMENT}-chatbot-echo-worker" \ + --start-time ${START_TIMESTAMP} \ + --end-time ${END_TIMESTAMP} \ + --region ${REGION} \ + --query-string ' +fields @timestamp, @initDuration +| filter @type = "REPORT" and ispresent(@initDuration) +| stats + count() as cold_starts, + avg(@initDuration) as avg_init_ms, + max(@initDuration) as max_init_ms +' > /tmp/worker-cold.json + +WORKER_COLD_ID=$(cat /tmp/worker-cold.json | jq -r '.queryId') +sleep 5 +aws logs get-query-results --query-id ${WORKER_COLD_ID} --region ${REGION} --output table + +echo "" +echo "========================================" +echo "CloudWatch Metrics (Lambda)" +echo "========================================" + +# 5. Concurrent Executions +echo_info "" +echo_info "5. Concurrent Executions" +echo_info "----------------------------------------" + +echo_info "Router Lambda:" +aws cloudwatch get-metric-statistics \ + --namespace AWS/Lambda \ + --metric-name ConcurrentExecutions \ + --dimensions Name=FunctionName,Value=laco-${ENVIRONMENT}-slack-router \ + --start-time $(date -u -d @$START_TIMESTAMP +%Y-%m-%dT%H:%M:%S 2>/dev/null || date -u -r $START_TIMESTAMP +%Y-%m-%dT%H:%M:%S) \ + --end-time $(date -u +%Y-%m-%dT%H:%M:%S) \ + --period 60 \ + --statistics Maximum,Average \ + --region ${REGION} \ + --output table + +echo_info "" +echo_info "Echo Worker Lambda:" +aws cloudwatch get-metric-statistics \ + --namespace AWS/Lambda \ + --metric-name ConcurrentExecutions \ + --dimensions Name=FunctionName,Value=laco-${ENVIRONMENT}-chatbot-echo-worker \ + --start-time $(date -u -d @$START_TIMESTAMP +%Y-%m-%dT%H:%M:%S 2>/dev/null || date -u -r $START_TIMESTAMP +%Y-%m-%dT%H:%M:%S) \ + --end-time $(date -u +%Y-%m-%dT%H:%M:%S) \ + --period 60 \ + --statistics Maximum,Average \ + --region ${REGION} \ + --output table + +echo_info "" +echo_info "6. Throttles" +echo_info "----------------------------------------" + +echo_info "Router Throttles:" +aws cloudwatch get-metric-statistics \ + --namespace AWS/Lambda \ + --metric-name Throttles \ + --dimensions Name=FunctionName,Value=laco-${ENVIRONMENT}-slack-router \ + --start-time $(date -u -d @$START_TIMESTAMP +%Y-%m-%dT%H:%M:%S 2>/dev/null || date -u -r $START_TIMESTAMP +%Y-%m-%dT%H:%M:%S) \ + --end-time $(date -u +%Y-%m-%dT%H:%M:%S) \ + --period 60 \ + --statistics Sum \ + --region ${REGION} \ + --output table + +echo_info "" +echo_info "Worker Throttles:" +aws cloudwatch get-metric-statistics \ + --namespace AWS/Lambda \ + --metric-name Throttles \ + --dimensions Name=FunctionName,Value=laco-${ENVIRONMENT}-chatbot-echo-worker \ + --start-time $(date -u -d @$START_TIMESTAMP +%Y-%m-%dT%H:%M:%S 2>/dev/null || date -u -r $START_TIMESTAMP +%Y-%m-%dT%H:%M:%S) \ + --end-time $(date -u +%Y-%m-%dT%H:%M:%S) \ + --period 60 \ + --statistics Sum \ + --region ${REGION} \ + --output table + +echo_info "" +echo_info "========================================" +echo_info "SQS Metrics" +echo_info "========================================" + +# 7. SQS Queue Metrics +echo_info "" +echo_info "7. SQS Queue Age" +echo_info "----------------------------------------" + +aws cloudwatch get-metric-statistics \ + --namespace AWS/SQS \ + --metric-name ApproximateAgeOfOldestMessage \ + --dimensions Name=QueueName,Value=laco-${ENVIRONMENT}-chatbot-echo \ + --start-time $(date -u -d @$START_TIMESTAMP +%Y-%m-%dT%H:%M:%S 2>/dev/null || date -u -r $START_TIMESTAMP +%Y-%m-%dT%H:%M:%S) \ + --end-time $(date -u +%Y-%m-%dT%H:%M:%S) \ + --period 60 \ + --statistics Average,Maximum \ + --region ${REGION} \ + --output table + +echo_info "" +echo_info "8. SQS Queue Depth" +echo_info "----------------------------------------" + +aws cloudwatch get-metric-statistics \ + --namespace AWS/SQS \ + --metric-name ApproximateNumberOfMessagesVisible \ + --dimensions Name=QueueName,Value=laco-${ENVIRONMENT}-chatbot-echo \ + --start-time $(date -u -d @$START_TIMESTAMP +%Y-%m-%dT%H:%M:%S 2>/dev/null || date -u -r $START_TIMESTAMP +%Y-%m-%dT%H:%M:%S) \ + --end-time $(date -u +%Y-%m-%dT%H:%M:%S) \ + --period 60 \ + --statistics Average,Maximum \ + --region ${REGION} \ + --output table + +echo_info "" +echo_info "========================================" +echo_info "Component-Level Breakdown (Estimated)" +echo_info "========================================" +echo_info "" +echo_info "Based on available metrics, estimated latency breakdown:" +echo_info "" +echo_info "β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”" +echo_info "β”‚ Component Flow β”‚ Estimated Time β”‚" +echo_info "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€" +echo_info "β”‚ 1. API Gateway β†’ Router Lambda β”‚ See section 1 β”‚" +echo_info "β”‚ 2. Router Lambda Processing β”‚ See section 1 β”‚" +echo_info "β”‚ 3. EventBridge β†’ SQS β†’ Worker β”‚ See section 7 β”‚" +echo_info "β”‚ (Queue Age) β”‚ (SQS Age) β”‚" +echo_info "β”‚ 4. Worker Lambda Processing β”‚ See section 2 β”‚" +echo_info "β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜" +echo_info "" +echo_info "Total E2E Latency (Estimated):" +echo_info " = Router Duration + SQS Age + Worker Duration" +echo_info "" +echo_info "Note: For more accurate per-component breakdown," +echo_info " consider adding timestamp tracking to Lambda code." +echo_info "" +echo_info "========================================" +echo_info "Analysis Complete" +echo_info "========================================" +echo_info "" +echo_info "Summary: Component performance analyzed" +echo_info "" +echo_info "Key Metrics to Check:" +echo_info " 1. Router Lambda P95 < 200ms (API Gateway processing)" +echo_info " 2. SQS Queue Age < 500ms (EventBridge + SQS delay)" +echo_info " 3. Worker Lambda P95 < 1500ms (Command processing)" +echo_info " 4. No throttles (Concurrency OK)" +echo_info " 5. Error rate < 1% (System stable)" +echo_info "" diff --git a/applications/chatops/slack-bot/performance-tests/artillery-config.yml b/applications/chatops/slack-bot/performance-tests/artillery-config.yml new file mode 100644 index 0000000..f78ba54 --- /dev/null +++ b/applications/chatops/slack-bot/performance-tests/artillery-config.yml @@ -0,0 +1,88 @@ +config: + target: "{{ $processEnvironment.API_GATEWAY_URL }}" + phases: + # Warm-up phase - ensure Lambdas are warm + - duration: 60 + arrivalRate: 2 + name: "Warm-up (2 req/s)" + + # Gradual ramp-up to test scaling + - duration: 180 + arrivalRate: 5 + rampTo: 20 + name: "Ramp-up (5 -> 20 req/s)" + + # Sustained load - test steady state performance + - duration: 300 + arrivalRate: 20 + name: "Sustained load (20 req/s)" + + # Approach API Gateway limit + - duration: 120 + arrivalRate: 20 + rampTo: 45 + name: "High load (20 -> 45 req/s)" + + # Cool-down + - duration: 60 + arrivalRate: 5 + name: "Cool-down (5 req/s)" + + processor: "./slack-signature-processor.js" + + plugins: + metrics-by-endpoint: + stripQueryString: true + metricsNamespace: "slack_bot_perf_test" + + # Performance thresholds + ensure: + maxErrorRate: 1 # Max 1% error rate + p95: 3000 # 95th percentile under 3 seconds + p99: 5000 # 99th percentile under 5 seconds + +scenarios: + # Weighted distribution matching realistic usage + - name: "Echo Command (Fast)" + weight: 40 + flow: + - post: + url: "/" + headers: + Content-Type: "application/x-www-form-urlencoded" + beforeRequest: "generateSlackSignature" + body: "token=test&team_id=T123&team_domain=test&channel_id=C123&channel_name=general&user_id=U{{ $randomNumber(1000, 9999) }}&user_name=testuser&command=/echo&text=performance test {{ $randomNumber(1, 10000) }}&response_url=https://hooks.slack.com/commands/T123/456/token&trigger_id=123.456.abc" + afterResponse: "captureMetrics" + + - name: "Status Command (Medium)" + weight: 30 + flow: + - post: + url: "/" + headers: + Content-Type: "application/x-www-form-urlencoded" + beforeRequest: "generateSlackSignature" + body: "token=test&team_id=T123&team_domain=test&channel_id=C123&channel_name=general&user_id=U{{ $randomNumber(1000, 9999) }}&user_name=testuser&command=/status&text=&response_url=https://hooks.slack.com/commands/T123/456/token&trigger_id=123.456.abc" + afterResponse: "captureMetrics" + + - name: "Deploy Command (Slow)" + weight: 20 + flow: + - post: + url: "/" + headers: + Content-Type: "application/x-www-form-urlencoded" + beforeRequest: "generateSlackSignature" + body: "token=test&team_id=T123&team_domain=test&channel_id=C123&channel_name=general&user_id=U{{ $randomNumber(1000, 9999) }}&user_name=testuser&command=/deploy&text=app-v{{ $randomNumber(1, 100) }}.0&response_url=https://hooks.slack.com/commands/T123/456/token&trigger_id=123.456.abc" + afterResponse: "captureMetrics" + + - name: "Build Command (Slow)" + weight: 10 + flow: + - post: + url: "/" + headers: + Content-Type: "application/x-www-form-urlencoded" + beforeRequest: "generateSlackSignature" + body: "token=test&team_id=T123&team_domain=test&channel_id=C123&channel_name=general&user_id=U{{ $randomNumber(1000, 9999) }}&user_name=testuser&command=/build&text=main&response_url=https://hooks.slack.com/commands/T123/456/token&trigger_id=123.456.abc" + afterResponse: "captureMetrics" diff --git a/applications/chatops/slack-bot/performance-tests/artillery-echo-light.yml b/applications/chatops/slack-bot/performance-tests/artillery-echo-light.yml new file mode 100644 index 0000000..1da48b7 --- /dev/null +++ b/applications/chatops/slack-bot/performance-tests/artillery-echo-light.yml @@ -0,0 +1,60 @@ +config: + target: "{{ $processEnvironment.API_GATEWAY_URL }}" + + # Enable console reporting + engines: + socketio: + maxPoolSize: 50 + + phases: + # Warm-up phase + - duration: 30 + arrivalRate: 2 + name: "Warm-up (2 req/s)" + + # Gradual ramp-up (API Gateway μ œν•œ κ³ λ €) + - duration: 120 + arrivalRate: 5 + rampTo: 20 + name: "Ramp-up (5 -> 20 req/s)" + + # Sustained load (μ•ˆμ „ν•œ λ²”μœ„) + - duration: 180 + arrivalRate: 20 + name: "Sustained load (20 req/s)" + + # Medium-high load (μ œν•œμ˜ 80%) + - duration: 60 + arrivalRate: 20 + rampTo: 40 + name: "High load (20 -> 40 req/s)" + + # Cool-down + - duration: 30 + arrivalRate: 5 + name: "Cool-down (5 req/s)" + + processor: "./slack-signature-processor.js" + + plugins: + metrics-by-endpoint: + stripQueryString: true + metricsNamespace: "slack_bot_echo_light" + + # Performance thresholds for echo command + ensure: + maxErrorRate: 1 # Max 1% error rate + p95: 2000 # 95th percentile under 2s + p99: 3000 # 99th percentile under 3s + +scenarios: + # Only echo command - API Gateway μ œν•œ κ³ λ € + - name: "Echo Command Performance Test (Light)" + flow: + - post: + url: "/" + headers: + Content-Type: "application/x-www-form-urlencoded" + beforeRequest: "generateSlackSignature" + body: "" # Body will be generated in beforeRequest hook + afterResponse: "captureMetrics" diff --git a/applications/chatops/slack-bot/performance-tests/artillery-echo-only.yml b/applications/chatops/slack-bot/performance-tests/artillery-echo-only.yml new file mode 100644 index 0000000..4c8c1b3 --- /dev/null +++ b/applications/chatops/slack-bot/performance-tests/artillery-echo-only.yml @@ -0,0 +1,54 @@ +config: + target: "{{ $processEnvironment.API_GATEWAY_URL }}" + phases: + # Warm-up phase + - duration: 30 + arrivalRate: 2 + name: "Warm-up (2 req/s)" + + # Gradual ramp-up + - duration: 120 + arrivalRate: 5 + rampTo: 30 + name: "Ramp-up (5 -> 30 req/s)" + + # Sustained load + - duration: 180 + arrivalRate: 30 + name: "Sustained load (30 req/s)" + + # High load test + - duration: 60 + arrivalRate: 30 + rampTo: 50 + name: "High load (30 -> 50 req/s)" + + # Cool-down + - duration: 30 + arrivalRate: 5 + name: "Cool-down (5 req/s)" + + processor: "./slack-signature-processor.js" + + plugins: + metrics-by-endpoint: + stripQueryString: true + metricsNamespace: "slack_bot_echo_test" + + # Performance thresholds for echo command + ensure: + maxErrorRate: 1 # Max 1% error rate + p95: 2000 # Echo should be faster - 95th percentile under 2s + p99: 3000 # 99th percentile under 3s + +scenarios: + # Only echo command - simplest, fastest processing + - name: "Echo Command Performance Test" + flow: + - post: + url: "/" + headers: + Content-Type: "application/x-www-form-urlencoded" + beforeRequest: "generateSlackSignature" + body: "token=test&team_id=T123&team_domain=test&channel_id=C123&channel_name=general&user_id=U{{ $randomNumber(1000, 9999) }}&user_name=testuser{{ $randomNumber(1, 100) }}&command=/echo&text=performance test message {{ $randomNumber(1, 100000) }}&response_url=https://hooks.slack.com/commands/T123/456/token&trigger_id=123.456.abc" + afterResponse: "captureMetrics" diff --git a/applications/chatops/slack-bot/performance-tests/render-report.js b/applications/chatops/slack-bot/performance-tests/render-report.js new file mode 100644 index 0000000..6cff8a8 --- /dev/null +++ b/applications/chatops/slack-bot/performance-tests/render-report.js @@ -0,0 +1,978 @@ +#!/usr/bin/env node +"use strict"; + +const fs = require("fs"); +const path = require("path"); + +function die(message) { + console.error(message); + process.exit(1); +} + +const inputPath = process.argv[2]; +if (!inputPath) { + die("Usage: node render-report.js [output.html]"); +} + +if (!fs.existsSync(inputPath)) { + die(`Input not found: ${inputPath}`); +} + +let outputPath = process.argv[3]; +if (!outputPath) { + outputPath = inputPath.endsWith(".json") + ? inputPath.slice(0, -5) + ".html" + : inputPath + ".html"; +} + +// .metrics.json 파일 경둜 생성 및 읽기 +let metricsData = null; +if (inputPath.endsWith(".json") && !inputPath.includes(".metrics.json")) { + const metricsPath = inputPath.slice(0, -5) + ".metrics.json"; + if (fs.existsSync(metricsPath)) { + try { + metricsData = JSON.parse(fs.readFileSync(metricsPath, "utf8")); + console.log(`βœ“ Loaded metrics from: ${metricsPath}`); + } catch (err) { + console.warn(`Warning: Failed to parse metrics: ${err.message}`); + } + } +} + +let raw; +try { + raw = JSON.parse(fs.readFileSync(inputPath, "utf8")); +} catch (err) { + die(`Failed to parse JSON: ${err.message}`); +} + +const aggregate = raw.aggregate || {}; +const counters = aggregate.counters || {}; +const summaries = aggregate.summaries || {}; +const histograms = aggregate.histograms || {}; +const intermediate = Array.isArray(raw.intermediate) ? raw.intermediate : []; + +const responseSummary = summaries["http.response_time"] || {}; +const sessionSummary = summaries["vusers.session_length"] || {}; +const responseHistogram = histograms["http.response_time"] || {}; + +const toNumber = (value) => (Number.isFinite(value) ? value : 0); + +const totalRequests = toNumber(counters["http.requests"]); +const totalResponses = toNumber(counters["http.responses"]); +const errorCount = toNumber(counters["errors.total"]); +const durationMs = toNumber(sessionSummary.max); +const avgRps = durationMs ? totalRequests / (durationMs / 1000) : 0; + +const httpCodes = {}; +const otherCodes = {}; +Object.entries(counters).forEach(([key, value]) => { + const match = key.match(/(?:^|\\.)codes\\.(\\d+)$/); + if (!match) return; + const code = match[1]; + if (key.startsWith("http.codes.")) { + httpCodes[code] = (httpCodes[code] || 0) + toNumber(value); + } else { + otherCodes[code] = (otherCodes[code] || 0) + toNumber(value); + } +}); + +const codesSource = Object.keys(httpCodes).length ? httpCodes : otherCodes; +const codes = Object.entries(codesSource) + .map(([code, count]) => ({ code, count })) + .sort((a, b) => Number(a.code) - Number(b.code)); + +const errorMap = {}; +Object.entries(counters).forEach(([key, value]) => { + const idx = key.lastIndexOf("errors."); + if (idx === -1) return; + const label = key.slice(idx + "errors.".length); + if (label === "total") return; + errorMap[label] = (errorMap[label] || 0) + toNumber(value); +}); + +const errors = Object.entries(errorMap) + .map(([label, count]) => ({ label, count })) + .sort((a, b) => b.count - a.count); + +const summary = { + requests: totalRequests, + responses: totalResponses, + errors: errorCount, + errorRate: totalRequests ? (errorCount / totalRequests) * 100 : 0, + durationMs, + avgRps, + min: toNumber(responseSummary.min), + max: toNumber(responseSummary.max), + mean: toNumber(responseSummary.mean), + median: toNumber(responseSummary.median || responseSummary.p50), + p95: toNumber(responseSummary.p95), + p99: toNumber(responseSummary.p99), + cloudwatch: metricsData ? metricsData.cloudwatch : null, +}; + +const vusers = { + created: toNumber(counters["vusers.created"]), + completed: toNumber(counters["vusers.completed"]), + failed: toNumber(counters["vusers.failed"]), + scenarios: Object.entries(counters) + .filter(([key]) => key.startsWith("vusers.created_by_name.")) + .map(([key, value]) => ({ + name: key.replace("vusers.created_by_name.", ""), + count: toNumber(value), + })) + .sort((a, b) => b.count - a.count), +}; + +summary.vusersCreated = vusers.created; +summary.vusersCompleted = vusers.completed; +summary.vusersFailed = vusers.failed; + +const endpoints = Object.entries(summaries) + .filter(([key]) => key.includes(".response_time.") && !key.startsWith("http.")) + .map(([key, stats]) => { + const label = key.split(".response_time.")[1] || key; + const histogram = histograms[key] || {}; + return { + label, + min: toNumber(stats.min), + max: toNumber(stats.max), + mean: toNumber(stats.mean), + median: toNumber(stats.median || stats.p50), + p95: toNumber(stats.p95 || histogram.p95), + p99: toNumber(stats.p99 || histogram.p99), + count: toNumber(stats.count), + }; + }) + .sort((a, b) => b.p95 - a.p95); + +const rawAggregate = { + counters, + summaries, + histograms, + rates: aggregate.rates || {}, + firstCounterAt: aggregate.firstCounterAt, + lastCounterAt: aggregate.lastCounterAt, + firstHistogramAt: aggregate.firstHistogramAt, + lastHistogramAt: aggregate.lastHistogramAt, + firstMetricAt: aggregate.firstMetricAt, + lastMetricAt: aggregate.lastMetricAt, + period: aggregate.period, +}; + +const series = intermediate + .map((item, index) => { + const stats = (item.summaries || {})["http.response_time"] || {}; + const rates = item.rates || {}; + const ts = item.period || item.firstMetricAt || index; + return { + ts, + median: toNumber(stats.median || stats.p50), + p95: toNumber(stats.p95), + p99: toNumber(stats.p99), + rps: toNumber(rates["http.request_rate"]), + }; + }) + .filter((point) => point.ts !== undefined); + +const percentileKeys = ["p50", "p75", "p90", "p95", "p99", "p999"]; +const percentiles = percentileKeys + .map((key) => ({ + label: key.toUpperCase(), + value: toNumber(responseHistogram[key]), + })) + .filter((item) => item.value > 0); + +const reportData = { + meta: { + source: path.basename(inputPath), + generatedAt: new Date().toISOString(), + }, + summary, + series, + percentiles, + codes, + errors, + endpoints, + scenarios: vusers.scenarios, + rawAggregate, +}; + +const html = ` + + + + + Artillery E2E Report + + + + + + +
+

Artillery E2E Report

+
+
+
+
+
+
Requests
+
+
+
+
Responses
+
+
+
+
VUsers Created
+
+
+
+
VUsers Completed
+
+
+
+
VUsers Failed
+
+
+
+
Errors
+
+
+
+
+
Avg RPS
+
+
+
+
Duration
+
+
+
+
P95 Latency
+
+
+
+
P99 Latency
+
+
+
+ +
+
+

Response Time (ms)

+
+ + + +
+ +
+
+

Request Rate (req/s)

+ +
+
+

Latency Percentiles (ms)

+ +
+
+

Status Codes

+
No status codes recorded.
+ +
+
+

Error Types

+
No errors recorded.
+ +
+
+ +
+
+

Endpoint Latency (ms)

+ + + + + + + + + + + +
EndpointMedianP95P99Count
+
+
+

Scenario VUsers

+ + + + + + + + +
ScenarioCreated
+
+
+ + + + +
+

Raw Aggregate Metrics

+
+ Counters + 

+        

+        

+          Summaries
+          

+        

+        

+          Histograms
+          

+        

+        

+          Rates
+          

+        

+        

+          Timing Metadata
+          

+        

+      

+      

+        Charts are rendered in-browser using Chart.js (CDN).
+      

+    

+
+    
+    
+  
+
+`;
+
+fs.writeFileSync(outputPath, html, "utf8");
+console.log(`Report written to ${outputPath}`);
diff --git a/applications/chatops/slack-bot/performance-tests/slack-signature-processor.js b/applications/chatops/slack-bot/performance-tests/slack-signature-processor.js
new file mode 100644
index 0000000..3a64df1
--- /dev/null
+++ b/applications/chatops/slack-bot/performance-tests/slack-signature-processor.js
@@ -0,0 +1,132 @@
+const crypto = require('crypto');
+const { SSMClient, GetParameterCommand } = require('@aws-sdk/client-ssm');
+
+let slackSigningSecret = null;
+let isFetchingSecret = false;
+let secretPromise = null;
+
+/**
+ * Fetch Slack signing secret from AWS SSM Parameter Store
+ * Cached after first retrieval for performance
+ */
+async function getSlackSigningSecret() {
+  // Return cached secret if available
+  if (slackSigningSecret) {
+    return slackSigningSecret;
+  }
+
+  // If already fetching, wait for that promise
+  if (isFetchingSecret && secretPromise) {
+    return secretPromise;
+  }
+
+  // Start fetching
+  isFetchingSecret = true;
+  secretPromise = (async () => {
+    try {
+      const environment = process.env.ENVIRONMENT || 'plt';
+      const region = process.env.AWS_REGION || 'ca-central-1';
+      const parameterName = `/laco/${environment}/aws/secrets/slack/signing-secret`;
+
+      console.log(`Fetching Slack signing secret from SSM: ${parameterName}`);
+
+      const ssm = new SSMClient({ region });
+      const response = await ssm.send(new GetParameterCommand({
+        Name: parameterName,
+        WithDecryption: true
+      }));
+
+      slackSigningSecret = response.Parameter.Value;
+      console.log('βœ“ Slack signing secret retrieved successfully');
+      return slackSigningSecret;
+    } catch (error) {
+      console.error('βœ— Failed to fetch Slack signing secret from SSM:', error.message);
+      console.error('  Make sure you have AWS credentials configured and access to Parameter Store');
+      throw error;
+    } finally {
+      isFetchingSecret = false;
+    }
+  })();
+
+  return secretPromise;
+}
+
+/**
+ * Generate Slack request signature
+ * See: https://api.slack.com/authentication/verifying-requests-from-slack
+ *
+ * IMPORTANT: Must be called AFTER Artillery processes templates
+ * This is a workaround since Artillery evaluates templates after beforeRequest hooks
+ */
+function generateSlackSignature(requestParams, context, ee, next) {
+  // Generate random values first (to replace Artillery templates)
+  const userId = Math.floor(Math.random() * (9999 - 1000 + 1)) + 1000;
+  const userName = Math.floor(Math.random() * 100) + 1;
+  const messageId = Math.floor(Math.random() * 100000) + 1;
+
+  // Build body with actual values (not templates)
+  const body = `token=test&team_id=T123&team_domain=test&channel_id=C123&channel_name=general&user_id=U${userId}&user_name=testuser${userName}&command=/echo&text=performance test message ${messageId}&response_url=https://hooks.slack.com/commands/T123/456/token&trigger_id=123.456.abc`;
+
+  // Replace requestParams.body with our pre-evaluated body
+  requestParams.body = body;
+
+  const timestamp = Math.floor(Date.now() / 1000);
+
+  getSlackSigningSecret()
+    .then(secret => {
+      // Create signature base string (must match exactly what Slack/Lambda expects)
+      const sigBasestring = `v0:${timestamp}:${body}`;
+
+      // Generate HMAC signature
+      const signature = 'v0=' + crypto
+        .createHmac('sha256', secret)
+        .update(sigBasestring)
+        .digest('hex');
+
+      // Add headers to request (case-sensitive for API Gateway)
+      requestParams.headers['X-Slack-Request-Timestamp'] = timestamp.toString();
+      requestParams.headers['X-Slack-Signature'] = signature;
+
+      // Debug log for first few requests
+      if (context.vars.$loopCount === undefined || context.vars.$loopCount < 3) {
+        console.log(`[DEBUG] Signature generated for timestamp ${timestamp}`);
+        console.log(`[DEBUG] Body length: ${body.length}`);
+        console.log(`[DEBUG] Body preview: ${body.substring(0, 100)}...`);
+        console.log(`[DEBUG] Signature: ${signature.substring(0, 20)}...`);
+      }
+
+      return next();
+    })
+    .catch(err => {
+      console.error('Failed to generate signature:', err);
+      return next(err);
+    });
+}
+
+/**
+ * Capture response metrics for analysis
+ */
+function captureMetrics(requestParams, response, context, ee, next) {
+  const statusCode = response.statusCode;
+  const responseTime = response.timings.phases.total;
+
+  // Log slow responses
+  if (responseTime > 3000) {
+    console.log(`⚠ Slow response: ${responseTime}ms (status: ${statusCode})`);
+  }
+
+  // Log errors
+  if (statusCode >= 400) {
+    console.log(`βœ— Error response: ${statusCode} (time: ${responseTime}ms)`);
+    if (response.body) {
+      console.log(`  Body: ${response.body.substring(0, 200)}`);
+    }
+  }
+
+  return next();
+}
+
+module.exports = {
+  generateSlackSignature,
+  captureMetrics
+};
diff --git a/applications/chatops/slack-bot/performance-tests/test-curl.sh b/applications/chatops/slack-bot/performance-tests/test-curl.sh
new file mode 100755
index 0000000..375d7ed
--- /dev/null
+++ b/applications/chatops/slack-bot/performance-tests/test-curl.sh
@@ -0,0 +1,93 @@
+#!/bin/bash
+# Test Slack signature with curl
+
+set -e
+
+# Check if API Gateway URL is provided
+if [ -z "$1" ]; then
+  echo "Usage: $0 "
+  echo ""
+  echo "Example:"
+  echo "  $0 https://xxxxxx.execute-api.ca-central-1.amazonaws.com/prod/slack"
+  echo ""
+  exit 1
+fi
+
+API_GATEWAY_URL="$1"
+ENVIRONMENT="${ENVIRONMENT:-plt}"
+
+echo "========================================"
+echo "Slack Signature Test with curl"
+echo "========================================"
+echo ""
+
+# Get signing secret from SSM
+echo "Fetching Slack signing secret..."
+SIGNING_SECRET=$(aws ssm get-parameter \
+  --name "/laco/${ENVIRONMENT}/aws/secrets/slack/signing-secret" \
+  --with-decryption \
+  --query 'Parameter.Value' \
+  --output text \
+  --region ca-central-1)
+
+if [ -z "$SIGNING_SECRET" ]; then
+  echo "ERROR: Could not fetch signing secret"
+  exit 1
+fi
+
+echo "βœ“ Signing secret retrieved"
+echo ""
+
+# Prepare request
+TIMESTAMP=$(date +%s)
+BODY="token=test&team_id=T123&team_domain=test&channel_id=C123&channel_name=general&user_id=U1234&user_name=testuser&command=/echo&text=test message&response_url=https://hooks.slack.com/test&trigger_id=123.456"
+
+echo "Request details:"
+echo "  Timestamp: $TIMESTAMP"
+echo "  Body: ${BODY:0:80}..."
+echo ""
+
+# Generate signature
+SIG_BASESTRING="v0:${TIMESTAMP}:${BODY}"
+SIGNATURE="v0=$(echo -n "$SIG_BASESTRING" | openssl dgst -sha256 -hmac "$SIGNING_SECRET" | awk '{print $2}')"
+
+echo "Generated signature: ${SIGNATURE:0:30}..."
+echo ""
+
+# Make request
+echo "Sending request to: ${API_GATEWAY_URL}"
+echo ""
+
+# Use --data-raw to prevent curl from encoding the body
+RESPONSE=$(curl -s -w "\nHTTP_STATUS:%{http_code}" \
+  -X POST "${API_GATEWAY_URL}" \
+  -H "Content-Type: application/x-www-form-urlencoded" \
+  -H "X-Slack-Request-Timestamp: ${TIMESTAMP}" \
+  -H "X-Slack-Signature: ${SIGNATURE}" \
+  --data-raw "$BODY")
+
+HTTP_STATUS=$(echo "$RESPONSE" | grep "HTTP_STATUS:" | cut -d: -f2)
+BODY_RESPONSE=$(echo "$RESPONSE" | sed '/HTTP_STATUS:/d')
+
+echo "========================================"
+echo "Response:"
+echo "========================================"
+echo "Status: $HTTP_STATUS"
+echo "Body: $BODY_RESPONSE"
+echo ""
+
+if [ "$HTTP_STATUS" == "200" ]; then
+  echo "βœ“ SUCCESS - Signature validation passed!"
+  exit 0
+elif [ "$HTTP_STATUS" == "401" ] || [ "$HTTP_STATUS" == "403" ]; then
+  echo "βœ— FAILED - Signature validation failed (401/403)"
+  echo ""
+  echo "Debugging info:"
+  echo "  - Check that signing secret is correct in SSM"
+  echo "  - Verify timestamp is not too old (within 5 minutes)"
+  echo "  - Ensure body format matches exactly what Lambda expects"
+  exit 1
+else
+  echo "βœ— FAILED - Unexpected status code: $HTTP_STATUS"
+  exit 1
+fi
diff --git a/applications/chatops/slack-bot/performance-tests/test-signature.js b/applications/chatops/slack-bot/performance-tests/test-signature.js
new file mode 100644
index 0000000..9c13fa9
--- /dev/null
+++ b/applications/chatops/slack-bot/performance-tests/test-signature.js
@@ -0,0 +1,47 @@
+// Test signature generation locally
+const crypto = require('crypto');
+
+// Simulated signing secret (replace with actual for testing)
+const signingSecret = 'test-secret-replace-with-real';
+
+// Simulate Artillery request
+const timestamp = Math.floor(Date.now() / 1000);
+const body = "token=test&team_id=T123&team_domain=test&channel_id=C123&channel_name=general&user_id=U1234&user_name=testuser1&command=/echo&text=performance test message 12345&response_url=https://hooks.slack.com/commands/T123/456/token&trigger_id=123.456.abc";
+
+console.log('='.repeat(60));
+console.log('Testing Slack Signature Generation');
+console.log('='.repeat(60));
+console.log('\n1. Request Details:');
+console.log('   Timestamp:', timestamp);
+console.log('   Body length:', body.length);
+console.log('   Body:', body.substring(0, 100) + '...');
+
+// Generate signature
+const sigBasestring = `v0:${timestamp}:${body}`;
+console.log('\n2. Signature Base String:');
+console.log('   ', sigBasestring.substring(0, 100) + '...');
+
+const signature = 'v0=' + crypto
+  .createHmac('sha256', signingSecret)
+  .update(sigBasestring)
+  .digest('hex');
+
+console.log('\n3. Generated Signature:');
+console.log('   ', signature);
+
+console.log('\n4. Headers to send:');
+console.log('   X-Slack-Request-Timestamp:', timestamp);
+console.log('   X-Slack-Signature:', signature);
+
+console.log('\n5. Test with curl:');
+console.log(`
+curl -X POST https://YOUR_API_GATEWAY_URL/slack \\
+  -H "Content-Type: application/x-www-form-urlencoded" \\
+  -H "X-Slack-Request-Timestamp: ${timestamp}" \\
+  -H "X-Slack-Signature: ${signature}" \\
+  -d '${body}'
+`);
+
+console.log('\n' + '='.repeat(60));
+console.log('Note: Replace signingSecret with actual secret from SSM');
+console.log('='.repeat(60));
diff --git a/applications/chatops/slack-bot/performance-tests/view-results.sh b/applications/chatops/slack-bot/performance-tests/view-results.sh
new file mode 100755
index 0000000..e5d5e44
--- /dev/null
+++ b/applications/chatops/slack-bot/performance-tests/view-results.sh
@@ -0,0 +1,140 @@
+#!/bin/bash
+# View Artillery test results in terminal
+
+# metrics.json 파일 μ œμ™Έν•˜κ³  μ‹€μ œ Artillery ν…ŒμŠ€νŠΈ 결과만 μ°ΎκΈ°
+LATEST_JSON=$(ls -t results/*-test-*.json 2>/dev/null | grep -v '\.metrics\.json' | head -n1)
+
+if [ -z "$LATEST_JSON" ]; then
+  echo "No test results found"
+  exit 1
+fi
+
+# λŒ€μ‘ν•˜λŠ” metrics.json 파일 확인
+METRICS_JSON="${LATEST_JSON%.json}.metrics.json"
+HAS_METRICS=0
+if [ -f "$METRICS_JSON" ]; then
+  HAS_METRICS=1
+fi
+
+echo "========================================"
+echo "Performance Test Results"
+echo "========================================"
+echo "File: $LATEST_JSON"
+if [ $HAS_METRICS -eq 1 ]; then
+  echo "Metrics: $METRICS_JSON (CloudWatch + E2E Data)"
+fi
+echo ""
+
+if ! command -v jq &> /dev/null; then
+  echo "⚠️  jq not installed. Showing raw summary..."
+  echo ""
+  cat "$LATEST_JSON" | grep -A 50 "aggregate"
+else
+  echo "Summary Statistics:"
+  echo "----------------------------------------"
+
+  # Overall stats
+  echo ""
+  echo "πŸ“Š Request Statistics:"
+  jq -r '.aggregate.counters | to_entries[] | "  \(.key): \(.value)"' "$LATEST_JSON"
+
+  echo ""
+  echo "⏱️  Response Time (ms):"
+  jq -r '.aggregate.summaries["http.response_time"] |
+    "  Min:    \(.min)",
+    "  Max:    \(.max)",
+    "  Median: \(.median)",
+    "  P95:    \(.p95)",
+    "  P99:    \(.p99)"' "$LATEST_JSON"
+
+  echo ""
+  echo "🎯 Latency Percentiles (ms):"
+  jq -r '.aggregate.histograms["http.response_time"] | to_entries[] |
+    select(.key | tonumber? != null) |
+    "  P\(.key): \(.value)"' "$LATEST_JSON" | sort -t: -k1 -V | head -10
+
+  echo ""
+  echo "πŸ“ˆ Throughput:"
+  DURATION=$(jq -r '.aggregate.summaries["vusers.session_length"].max // 0' "$LATEST_JSON")
+  REQUESTS=$(jq -r '.aggregate.counters["http.requests"] // 0' "$LATEST_JSON")
+  if [ "$DURATION" != "0" ]; then
+    RPS=$(echo "scale=2; $REQUESTS / ($DURATION / 1000)" | bc)
+    echo "  Requests: $REQUESTS"
+    echo "  Duration: ${DURATION}ms"
+    echo "  Avg RPS:  $RPS req/s"
+  fi
+
+  echo ""
+  echo "❌ Error Codes:"
+  jq -r '.aggregate.codes // {} | to_entries[] | "  \(.key): \(.value)"' "$LATEST_JSON"
+
+  echo ""
+  echo "πŸ” Errors:"
+  ERROR_COUNT=$(jq -r '.aggregate.counters["errors.total"] // 0' "$LATEST_JSON")
+  if [ "$ERROR_COUNT" != "0" ]; then
+    jq -r '.aggregate.errors // {} | to_entries[] | "  \(.key): \(.value)"' "$LATEST_JSON"
+  else
+    echo "  No errors"
+  fi
+
+  echo ""
+  echo "========================================"
+
+  # Check thresholds
+  echo ""
+  echo "Threshold Check:"
+  echo "----------------------------------------"
+
+  P95=$(jq -r '.aggregate.summaries["http.response_time"].p95 // 0' "$LATEST_JSON")
+  P99=$(jq -r '.aggregate.summaries["http.response_time"].p99 // 0' "$LATEST_JSON")
+  ERROR_RATE=$(echo "scale=4; $ERROR_COUNT * 100 / $REQUESTS" | bc 2>/dev/null || echo "0")
+
+  echo "  P95 < 2000ms:  $P95 ms $([ $(echo "$P95 < 2000" | bc) -eq 1 ] && echo "βœ“" || echo "βœ—")"
+  echo "  P99 < 3000ms:  $P99 ms $([ $(echo "$P99 < 3000" | bc) -eq 1 ] && echo "βœ“" || echo "βœ—")"
+  echo "  Error < 1%:    ${ERROR_RATE}% $([ $(echo "$ERROR_RATE < 1" | bc) -eq 1 ] && echo "βœ“" || echo "βœ—")"
+
+  # CloudWatch Metricsκ°€ 있으면 ν‘œμ‹œ
+  if [ $HAS_METRICS -eq 1 ]; then
+    echo ""
+    echo "========================================"
+    echo "CloudWatch & E2E Metrics"
+    echo "========================================"
+
+    echo ""
+    echo "πŸ“Š Router Lambda (API Gateway β†’ Router):"
+    echo "----------------------------------------"
+    jq -r '.cloudwatch.router | "  Invocations: \(.invocations)\n  Avg:         \(.avg_ms) ms\n  P50:         \(.p50_ms) ms\n  P95:         \(.p95_ms) ms\n  P99:         \(.p99_ms) ms\n  Max:         \(.max_ms) ms"' "$METRICS_JSON"
+
+    echo ""
+    echo "βš™οΈ  Worker Lambda (EventBridge β†’ SQS β†’ Worker):"
+    echo "----------------------------------------"
+    jq -r '.cloudwatch.worker | "  Invocations: \(.invocations)\n  Avg:         \(.avg_ms) ms\n  P50:         \(.p50_ms) ms\n  P95:         \(.p95_ms) ms\n  P99:         \(.p99_ms) ms\n  Max:         \(.max_ms) ms"' "$METRICS_JSON"
+
+    echo ""
+    echo "πŸ”„ End-to-End (API Gateway β†’ Worker Lambda Completion):"
+    echo "----------------------------------------"
+    E2E_DATA=$(jq -r '.cloudwatch.e2e // {} | length' "$METRICS_JSON")
+    if [ "$E2E_DATA" -eq 0 ]; then
+      echo "  ⚠️  No E2E data available (Router not propagating correlation IDs)"
+    else
+      jq -r '.cloudwatch.e2e | "  Invocations: \(.invocations)\n  Avg:         \(.avg_ms) ms\n  P50:         \(.p50_ms) ms\n  P95:         \(.p95_ms) ms\n  P99:         \(.p99_ms) ms\n  Max:         \(.max_ms) ms"' "$METRICS_JSON"
+    fi
+
+    echo ""
+    echo "❌ Errors:"
+    echo "----------------------------------------"
+    jq -r '.cloudwatch.errors | "  Router:      \(.router)\n  Worker:      \(.worker)"' "$METRICS_JSON"
+
+    echo ""
+    echo "⏱️  Test Time Range:"
+    echo "----------------------------------------"
+    jq -r '.timeRange | "  Start:       \((.startMs / 1000 | floor) | todate)\n  End:         \((.endMs / 1000 | floor) | todate)\n  Duration:    \(.durationMs) ms (\((.durationMs / 1000) | floor) sec)"' "$METRICS_JSON"
+  fi
+
+fi
+
+echo ""
+echo "========================================"
+echo ""
+echo "πŸ’‘ Tip: Use 'make perf-test-analyze-test' to update CloudWatch metrics"
+echo ""

From b711dccecb7e7d2539cebf80ce6d9f2a156c84b6 Mon Sep 17 00:00:00 2001
From: Hyunseok Seo 
Date: Thu, 1 Jan 2026 23:21:56 -0500
Subject: [PATCH 03/15] Add E2E performance tracking to Slack bot echo worker
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Add correlation_id and api_gateway_start_time to SlackCommand and WorkerMessage types
- Track request start time in router Lambda (API Gateway entry point)
- Calculate and log comprehensive performance metrics in echo worker:
  - totalE2eMs: End-to-end latency from API Gateway to worker completion
  - workerDurationMs: Worker processing time
  - queueWaitMs: SQS queue wait time (difference between E2E and worker duration)
  - syncResponseMs: Synchronous Slack response time
  - asyncResponseMs: Asynchronous Slack response time
- Enable CloudWatch Insights analysis of latency breakdown across system components

πŸ€– Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude 
---
 .../chatops/slack-bot/src/router/index.ts     | 28 ++++++++++++++++---
 .../chatops/slack-bot/src/shared/types.ts     |  3 ++
 .../slack-bot/src/workers/echo/index.ts       | 17 +++++++++++
 3 files changed, 44 insertions(+), 4 deletions(-)

diff --git a/applications/chatops/slack-bot/src/router/index.ts b/applications/chatops/slack-bot/src/router/index.ts
index 5dd21c3..ea38642 100644
--- a/applications/chatops/slack-bot/src/router/index.ts
+++ b/applications/chatops/slack-bot/src/router/index.ts
@@ -19,19 +19,34 @@ const eventBridgeClient = new EventBridgeClient({
 });
 
 export async function handler(event: APIGatewayProxyEvent): Promise {
+  const apiGatewayStartTime = Date.now();
+
   logger.info('Router Lambda invoked', {
     path: event.path,
-    httpMethod: event.httpMethod
+    httpMethod: event.httpMethod,
+    startTime: apiGatewayStartTime
   });
 
   try {
+    // Debug: Log request details for signature troubleshooting
+    const timestamp = event.headers['x-slack-request-timestamp'] || event.headers['X-Slack-Request-Timestamp'] || '';
+    const signature = event.headers['x-slack-signature'] || event.headers['X-Slack-Signature'] || '';
+
+    logger.info('Request signature details', {
+      timestamp,
+      signaturePreview: signature.substring(0, 20) + '...',
+      bodyLength: (event.body || '').length,
+      bodyPreview: (event.body || '').substring(0, 100),
+      headerKeys: Object.keys(event.headers)
+    });
+
     // 1. Verify Slack signature
     const signingSecret = await getSlackSigningSecret();
     const isValid = await verifySlackSignature(
       signingSecret,
       {
-        'x-slack-request-timestamp': event.headers['x-slack-request-timestamp'] || event.headers['X-Slack-Request-Timestamp'] || '',
-        'x-slack-signature': event.headers['x-slack-signature'] || event.headers['X-Slack-Signature'] || ''
+        'x-slack-request-timestamp': timestamp,
+        'x-slack-signature': signature
       },
       event.body || ''
     );
@@ -46,6 +61,9 @@ export async function handler(event: APIGatewayProxyEvent): Promise {
         subsegment?.close();
 
         const totalDuration = Date.now() - startTime;
+        const e2eDuration = message.api_gateway_start_time
+          ? Date.now() - message.api_gateway_start_time
+          : undefined;
+
+        // Log structured performance metrics for CloudWatch Insights analysis
+        if (e2eDuration && correlationId) {
+          const queueWaitMs = e2eDuration - totalDuration;
+          logger.info('Performance metrics', {
+            correlationId,
+            totalE2eMs: e2eDuration,
+            workerDurationMs: totalDuration,
+            queueWaitMs: Math.max(0, queueWaitMs),
+            syncResponseMs: syncDuration,
+            asyncResponseMs: asyncDuration
+          });
+        }
 
         messageLogger.info('Echo command processed successfully', {
           totalDuration,
+          e2eDuration,
         });
       } catch (error) {
         subsegment?.addError(error as Error);

From 9a5577a891f7a98857177e3c4461dc372169c33c Mon Sep 17 00:00:00 2001
From: Hyunseok Seo 
Date: Thu, 1 Jan 2026 23:25:14 -0500
Subject: [PATCH 04/15] Update performance test scripts to use new structured
 metrics
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Update analyze-performance.sh to query Performance metrics log
  - Replace correlation-based E2E calculation with structured metrics
  - Add component breakdown: queueWaitMs, workerDurationMs, syncResponseMs, asyncResponseMs
  - Update summary table to show actual metrics instead of estimates
  - Adjust Key Metrics thresholds based on real data

- Update analyze-e2e-json.sh to extract all performance metrics
  - Add syncResponseMs and asyncResponseMs to E2E query
  - Add p50 percentile for better distribution analysis
  - Filter by 'Performance metrics' message for accurate data

Both scripts now leverage the structured performance logging added in previous commit.

πŸ€– Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude 
---
 .../performance-tests/analyze-e2e-json.sh     |  6 +-
 .../performance-tests/analyze-performance.sh  | 75 ++++++++++---------
 2 files changed, 41 insertions(+), 40 deletions(-)

diff --git a/applications/chatops/slack-bot/performance-tests/analyze-e2e-json.sh b/applications/chatops/slack-bot/performance-tests/analyze-e2e-json.sh
index 51539cc..6ae47c3 100755
--- a/applications/chatops/slack-bot/performance-tests/analyze-e2e-json.sh
+++ b/applications/chatops/slack-bot/performance-tests/analyze-e2e-json.sh
@@ -75,11 +75,11 @@ WORKER_METRICS=$(query_logs \
   "fields @duration | filter @type = \"REPORT\" | stats count() as invocations, avg(@duration) as avg_ms, percentile(@duration, 50) as p50_ms, percentile(@duration, 95) as p95_ms, percentile(@duration, 99) as p99_ms, max(@duration) as max_ms" \
   "worker")
 
-# 3. End-to-End Latency (if available)
-echo "Querying E2E latency..." >&2
+# 3. End-to-End Latency & Component Breakdown (from Performance metrics)
+echo "Querying E2E latency & component breakdown..." >&2
 E2E_METRICS=$(query_logs \
   "/aws/lambda/laco-${ENVIRONMENT}-chatbot-echo-worker" \
-  "fields totalE2eMs, queueWaitMs, workerDurationMs | filter totalE2eMs > 0 | stats count() as requests, avg(totalE2eMs) as avg_e2e_ms, percentile(totalE2eMs, 95) as p95_e2e_ms, percentile(totalE2eMs, 99) as p99_e2e_ms, avg(queueWaitMs) as avg_queue_ms" \
+  "fields totalE2eMs, queueWaitMs, workerDurationMs, syncResponseMs, asyncResponseMs | filter message = \"Performance metrics\" | stats count() as requests, avg(totalE2eMs) as avg_e2e_ms, percentile(totalE2eMs, 50) as p50_e2e_ms, percentile(totalE2eMs, 95) as p95_e2e_ms, percentile(totalE2eMs, 99) as p99_e2e_ms, avg(queueWaitMs) as avg_queue_wait_ms, avg(workerDurationMs) as avg_worker_ms, avg(syncResponseMs) as avg_sync_response_ms, avg(asyncResponseMs) as avg_async_response_ms" \
   "e2e" 2>/dev/null || echo "[]")
 
 # 4. Error Analysis
diff --git a/applications/chatops/slack-bot/performance-tests/analyze-performance.sh b/applications/chatops/slack-bot/performance-tests/analyze-performance.sh
index ae3a603..df51596 100755
--- a/applications/chatops/slack-bot/performance-tests/analyze-performance.sh
+++ b/applications/chatops/slack-bot/performance-tests/analyze-performance.sh
@@ -107,30 +107,31 @@ echo_info "========================================"
 echo_info "Environment: ${ENVIRONMENT}"
 echo_info ""
 
-# 0. End-to-End Latency (if correlation ID is present)
-echo_info "0. End-to-End Latency (Router β†’ Worker)"
+# 0. End-to-End Latency & Component Breakdown (from Performance metrics log)
+echo_info "0. End-to-End Latency & Component Breakdown"
 echo_info "----------------------------------------"
-echo_info "Note: Requires correlation ID tracking in logs"
+echo_info "Note: Using structured Performance metrics from echo worker"
 echo_info ""
 aws logs start-query \
-  --log-group-names "/aws/lambda/laco-${ENVIRONMENT}-slack-router" "/aws/lambda/laco-${ENVIRONMENT}-chatbot-echo-worker" \
+  --log-group-name "/aws/lambda/laco-${ENVIRONMENT}-chatbot-echo-worker" \
   --start-time ${START_TIMESTAMP} \
   --end-time ${END_TIMESTAMP} \
   --region ${REGION} \
   --query-string '
-fields @timestamp, @message, @logStream
-| filter @message like /correlationId/
-| parse @message "*correlationId*:*\"*\"*" as prefix, key, correlationId, suffix
-| stats earliest(@timestamp) as start, latest(@timestamp) as end by correlationId
-| filter isPresent(start) and isPresent(end)
-| fields correlationId, (end - start) as e2e_latency_ms
+fields @timestamp, totalE2eMs, workerDurationMs, queueWaitMs, syncResponseMs, asyncResponseMs
+| filter message = "Performance metrics"
 | stats
     count() as requests,
-    avg(e2e_latency_ms) as avg_e2e_ms,
-    percentile(e2e_latency_ms, 50) as p50_e2e_ms,
-    percentile(e2e_latency_ms, 95) as p95_e2e_ms,
-    percentile(e2e_latency_ms, 99) as p99_e2e_ms
-' > /tmp/e2e-query.json 2>/dev/null || echo "  ⚠ E2E tracking not available (correlation ID not found in logs)"
+    avg(totalE2eMs) as avg_e2e_ms,
+    percentile(totalE2eMs, 50) as p50_e2e_ms,
+    percentile(totalE2eMs, 95) as p95_e2e_ms,
+    percentile(totalE2eMs, 99) as p99_e2e_ms,
+    max(totalE2eMs) as max_e2e_ms,
+    avg(workerDurationMs) as avg_worker_ms,
+    avg(queueWaitMs) as avg_queue_wait_ms,
+    avg(syncResponseMs) as avg_sync_response_ms,
+    avg(asyncResponseMs) as avg_async_response_ms
+' > /tmp/e2e-query.json 2>/dev/null || echo "  ⚠ E2E tracking not available (Performance metrics not found in logs)"
 
 if [ -f /tmp/e2e-query.json ]; then
   E2E_QUERY_ID=$(cat /tmp/e2e-query.json | jq -r '.queryId' 2>/dev/null)
@@ -393,37 +394,37 @@ aws cloudwatch get-metric-statistics \
 
 echo_info ""
 echo_info "========================================"
-echo_info "Component-Level Breakdown (Estimated)"
+echo_info "Component-Level Breakdown"
 echo_info "========================================"
 echo_info ""
-echo_info "Based on available metrics, estimated latency breakdown:"
-echo_info ""
-echo_info "β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”"
-echo_info "β”‚ Component Flow                    β”‚ Estimated Time  β”‚"
-echo_info "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€"
-echo_info "β”‚ 1. API Gateway β†’ Router Lambda    β”‚ See section 1   β”‚"
-echo_info "β”‚ 2. Router Lambda Processing       β”‚ See section 1   β”‚"
-echo_info "β”‚ 3. EventBridge β†’ SQS β†’ Worker     β”‚ See section 7   β”‚"
-echo_info "β”‚    (Queue Age)                    β”‚ (SQS Age)       β”‚"
-echo_info "β”‚ 4. Worker Lambda Processing       β”‚ See section 2   β”‚"
-echo_info "β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜"
+echo_info "Latency breakdown from Performance metrics (Section 0):"
 echo_info ""
-echo_info "Total E2E Latency (Estimated):"
-echo_info "  = Router Duration + SQS Age + Worker Duration"
+echo_info "β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”"
+echo_info "β”‚ Component Flow                    β”‚ Metric             β”‚"
+echo_info "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€"
+echo_info "β”‚ 1. API Gateway β†’ Router Lambda    β”‚ See section 1      β”‚"
+echo_info "β”‚ 2. Router β†’ EventBridge β†’ SQS     β”‚ avg_queue_wait_ms  β”‚"
+echo_info "β”‚ 3. Worker Lambda Processing       β”‚ avg_worker_ms      β”‚"
+echo_info "β”‚    β”œβ”€ Sync Response               β”‚ avg_sync_resp_ms   β”‚"
+echo_info "β”‚    └─ Async Response              β”‚ avg_async_resp_ms  β”‚"
+echo_info "β”‚ Total E2E (API Gateway β†’ Done)    β”‚ avg_e2e_ms         β”‚"
+echo_info "β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜"
 echo_info ""
-echo_info "Note: For more accurate per-component breakdown,"
-echo_info "      consider adding timestamp tracking to Lambda code."
+echo_info "Formula:"
+echo_info "  totalE2eMs = workerDurationMs + queueWaitMs"
+echo_info "  queueWaitMs = Router processing + EventBridge + SQS polling"
 echo_info ""
 echo_info "========================================"
 echo_info "Analysis Complete"
 echo_info "========================================"
 echo_info ""
-echo_info "Summary: Component performance analyzed"
+echo_info "Summary: Component performance analyzed with structured metrics"
 echo_info ""
 echo_info "Key Metrics to Check:"
-echo_info "  1. Router Lambda P95 < 200ms  (API Gateway processing)"
-echo_info "  2. SQS Queue Age < 500ms      (EventBridge + SQS delay)"
-echo_info "  3. Worker Lambda P95 < 1500ms (Command processing)"
-echo_info "  4. No throttles               (Concurrency OK)"
-echo_info "  5. Error rate < 1%            (System stable)"
+echo_info "  1. E2E P95 < 3000ms           (Total user experience)"
+echo_info "  2. Queue Wait avg < 1000ms    (Router + EventBridge + SQS)"
+echo_info "  3. Worker P95 < 2500ms        (Command processing)"
+echo_info "  4. Sync Response < 500ms      (First Slack response)"
+echo_info "  5. No throttles               (Concurrency OK)"
+echo_info "  6. Error rate < 1%            (System stable)"
 echo_info ""

From 6cc35ad6dec4d469c44dc10783470897280cd26d Mon Sep 17 00:00:00 2001
From: Hyunseok Seo 
Date: Fri, 2 Jan 2026 11:13:35 -0500
Subject: [PATCH 05/15] Add error tracking to performance metrics
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Create logRouterMetrics() helper function for router Lambda
  - Track statusCode, duration, success/failure
  - Log authentication errors (401) with error type
  - Log server errors (500) with error details
  - Log successful requests (200) with command info

- Create logWorkerMetrics() helper function for worker Lambda
  - Add 'success' boolean field to all performance metrics
  - Add 'errorType' and 'errorMessage' for failed requests
  - Log metrics even when processing fails
  - Always include correlationId and command when available

- Enable CloudWatch Insights queries for error analysis:
  - Error rate calculation: count(success=false) / count(*)
  - Error type distribution
  - Performance comparison: success vs failure cases
  - Router vs Worker error breakdown

Example queries enabled:
- fields success, errorType | filter message = "Performance metrics" | stats count() by success, errorType
- fields statusCode, duration | filter message = "Router performance metrics" | stats avg(duration) by statusCode

πŸ€– Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude 
---
 .../chatops/slack-bot/src/router/index.ts     | 61 ++++++++++++++++++-
 .../slack-bot/src/workers/echo/index.ts       | 51 ++++++++++++----
 2 files changed, 98 insertions(+), 14 deletions(-)

diff --git a/applications/chatops/slack-bot/src/router/index.ts b/applications/chatops/slack-bot/src/router/index.ts
index ea38642..3f6097c 100644
--- a/applications/chatops/slack-bot/src/router/index.ts
+++ b/applications/chatops/slack-bot/src/router/index.ts
@@ -18,6 +18,31 @@ const eventBridgeClient = new EventBridgeClient({
   })
 });
 
+/**
+ * Log router performance metrics for monitoring and analysis
+ */
+function logRouterMetrics(params: {
+  statusCode: number;
+  duration: number;
+  correlationId?: string;
+  command?: string;
+  errorType?: string;
+  errorMessage?: string;
+}) {
+  const { statusCode, duration, correlationId, command, errorType, errorMessage } = params;
+  const success = statusCode >= 200 && statusCode < 300;
+
+  logger.info('Router performance metrics', {
+    correlationId,
+    command,
+    statusCode,
+    duration,
+    success,
+    ...(errorType && { errorType }),
+    ...(errorMessage && { errorMessage })
+  });
+}
+
 export async function handler(event: APIGatewayProxyEvent): Promise {
   const apiGatewayStartTime = Date.now();
 
@@ -52,7 +77,20 @@ export async function handler(event: APIGatewayProxyEvent): Promise {
   logger.info('Echo worker invoked', {
     recordCount: event.Records.length,
@@ -100,17 +118,16 @@ export async function handler(event: SQSEvent): Promise {
           : undefined;
 
         // Log structured performance metrics for CloudWatch Insights analysis
-        if (e2eDuration && correlationId) {
-          const queueWaitMs = e2eDuration - totalDuration;
-          logger.info('Performance metrics', {
-            correlationId,
-            totalE2eMs: e2eDuration,
-            workerDurationMs: totalDuration,
-            queueWaitMs: Math.max(0, queueWaitMs),
-            syncResponseMs: syncDuration,
-            asyncResponseMs: asyncDuration
-          });
-        }
+        logWorkerMetrics({
+          correlationId,
+          command: message.command,
+          totalE2eMs: e2eDuration,
+          workerDurationMs: totalDuration,
+          queueWaitMs: e2eDuration ? Math.max(0, e2eDuration - totalDuration) : undefined,
+          syncResponseMs: syncDuration,
+          asyncResponseMs: asyncDuration,
+          success: true
+        });
 
         messageLogger.info('Echo command processed successfully', {
           totalDuration,
@@ -123,12 +140,22 @@ export async function handler(event: SQSEvent): Promise {
       }
     } catch (error) {
       const duration = Date.now() - startTime;
+      const err = error as Error;
 
-      messageLogger.error('Failed to process echo command', error as Error, {
+      messageLogger.error('Failed to process echo command', err, {
         messageId: record.messageId,
         duration,
       });
 
+      // Log performance metrics even for failures
+      logWorkerMetrics({
+        correlationId,
+        workerDurationMs: duration,
+        success: false,
+        errorType: err.name,
+        errorMessage: err.message
+      });
+
       // Add to failed items for retry
       batchItemFailures.push({ itemIdentifier: record.messageId });
     }

From d039e5a696f51a31a0fbafcda2d487eb90039cbd Mon Sep 17 00:00:00 2001
From: Hyunseok Seo 
Date: Fri, 2 Jan 2026 12:12:14 -0500
Subject: [PATCH 06/15] Refactor Makefile perf-test targets and add minimal
 test profile
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Makefile improvements:
- Consolidate 11 perf-test targets into 5 clean commands
- Add PERF_PROFILE variable (minimal|light|full) with default=minimal
- Simplify command names: perf-test, perf-analyze, perf-summary, perf-report, perf-clean
- Fix perf-analyze-quiet to output JSON metrics file instead of suppressing everything
- Reduce code duplication by 60% (116 lines β†’ 64 lines)

New minimal test profile (artillery-echo-minimal.yml):
- Duration: 60 seconds (vs 420s for light)
- Requests: ~121 (vs ~5,460 for light)
- Cost reduction: 97.8%
- Provides statistically valid P50/P95/P99 metrics
- Perfect for quick validation and CI/CD

Performance script improvements:
- Skip CloudWatch Metrics in quiet mode (prevents failures)
- Output only essential progress messages in quiet mode
- Generate .metrics.json file for programmatic access

Usage examples:
  make perf-test                    # Run minimal (1 min, 121 reqs)
  make perf-test PERF_PROFILE=light # Run light (7 min, 5,460 reqs)
  make perf-test PERF_PROFILE=full  # Run full (12 min, all commands)
  make perf-analyze                 # Analyze with full output
  make perf-analyze-quiet           # Analyze quietly, save to JSON
  make perf-summary                 # Quick summary

πŸ€– Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude 
---
 applications/chatops/slack-bot/Makefile       | 163 ++++++------------
 .../performance-tests/analyze-performance.sh  |  15 +-
 .../artillery-echo-minimal.yml                |  49 ++++++
 3 files changed, 111 insertions(+), 116 deletions(-)
 create mode 100644 applications/chatops/slack-bot/performance-tests/artillery-echo-minimal.yml

diff --git a/applications/chatops/slack-bot/Makefile b/applications/chatops/slack-bot/Makefile
index 01ed22a..d5f66f9 100644
--- a/applications/chatops/slack-bot/Makefile
+++ b/applications/chatops/slack-bot/Makefile
@@ -49,16 +49,13 @@ help:
 	@echo "  make deploy-build           - Deploy build worker from local (default)"
 	@echo ""
 	@echo "$(GREEN)Performance Testing:$(NC)"
-	@echo "  make perf-test              - Full test (~12 min, all commands)"
-	@echo "  make perf-test-echo         - Echo-only test (~7 min)"
-	@echo "  make perf-test-quick        - Quick test (2 min)"
-	@echo "  make perf-test-results      - View latest test results (terminal)"
-	@echo "  make perf-test-analyze      - Analyze CloudWatch metrics (last 15 min)"
-	@echo "  make perf-test-analyze-test - Analyze from latest Artillery test"
-	@echo "  make perf-test-analyze-test-quiet - Analyze from latest test (auto mode)"
-	@echo "  make perf-test-json         - Export metrics to JSON (dashboard)"
-	@echo "  make perf-test-summary      - Quick summary of latest test"
-	@echo "  make perf-test-clean        - Clean test results"
+	@echo "  make perf-test              - Run test (default: minimal ~1min)"
+	@echo "    Options: PROFILE=minimal|light|full"
+	@echo "  make perf-analyze           - Analyze latest test results"
+	@echo "  make perf-analyze-quiet     - Analyze (quiet mode, for CI/CD)"
+	@echo "  make perf-summary           - Quick summary"
+	@echo "  make perf-report            - Generate HTML report"
+	@echo "  make perf-clean             - Clean test results"
 	@echo ""
 	@echo "$(GREEN)Deploy Lambda (Advanced):$(NC)"
 	@echo "  make deploy--local - Deploy from local dist/ (fast, no upload)"
@@ -241,116 +238,58 @@ clean-all: clean
 # Performance Testing
 # -----------------------------------------------------------------------------
 
-perf-test: perf-test-install
-	@echo "$(BLUE)Running Artillery performance test...$(NC)"
-	@echo "$(YELLOW)Environment: $(ENVIRONMENT)$(NC)"
-	@echo "$(YELLOW)Region: $(REGION)$(NC)"
-	@echo ""
-	@if [ -z "$(API_GATEWAY_URL)" ]; then \
-		echo "$(YELLOW)⚠ API_GATEWAY_URL not set. Getting from Terragrunt...$(NC)"; \
-		export API_GATEWAY_URL=$$(cd $(SANDBOX_ROOT)/slack-api-gateway && terragrunt output -raw api_gateway_url 2>/dev/null || echo ""); \
-		if [ -z "$$API_GATEWAY_URL" ]; then \
-			echo "$(YELLOW)βœ— Could not get API Gateway URL from Terragrunt$(NC)"; \
-			echo "$(YELLOW)  Please set manually: make perf-test API_GATEWAY_URL=https://xxx.execute-api.ca-central-1.amazonaws.com$(NC)"; \
-			exit 1; \
-		fi; \
-		echo "$(GREEN)βœ“ API Gateway URL: $$API_GATEWAY_URL$(NC)"; \
-		export ENVIRONMENT=$(ENVIRONMENT) AWS_REGION=$(REGION); \
-		cd performance-tests && artillery run artillery-config.yml --output results/test-$$(date +%Y%m%d-%H%M%S).json; \
-	else \
-		export ENVIRONMENT=$(ENVIRONMENT) AWS_REGION=$(REGION); \
-		cd performance-tests && artillery run artillery-config.yml --output results/test-$$(date +%Y%m%d-%H%M%S).json; \
+# Test profile configuration
+PERF_PROFILE ?= minimal
+PERF_CONFIG_minimal = artillery-echo-minimal.yml
+PERF_CONFIG_light = artillery-echo-light.yml
+PERF_CONFIG_full = artillery-config.yml
+
+perf-test-install:
+	@if ! command -v artillery > /dev/null; then \
+		echo "$(BLUE)Installing Artillery...$(NC)"; \
+		npm install -g artillery artillery-plugin-metrics-by-endpoint; \
 	fi
+	@cd performance-tests && npm install --no-save @aws-sdk/client-ssm 2>/dev/null || true
 
-perf-test-echo: perf-test-install
-	@echo "$(BLUE)Running Echo-only performance test (7 minutes, max 40 req/s)...$(NC)"
-	@echo "$(YELLOW)Environment: $(ENVIRONMENT)$(NC)"
-	@echo "$(YELLOW)Region: $(REGION)$(NC)"
-	@echo "$(YELLOW)⚠ Max load: 40 req/s (API Gateway limit: 50 req/s)$(NC)"
-	@echo ""
+perf-test: perf-test-install
+	@echo "$(BLUE)Running performance test [$(PERF_PROFILE)]...$(NC)"
 	@if [ -z "$(API_GATEWAY_URL)" ]; then \
-		echo "$(YELLOW)⚠ API_GATEWAY_URL not set. Getting from Terragrunt...$(NC)"; \
-		export API_GATEWAY_URL=$$(cd $(SANDBOX_ROOT)/slack-api-gateway && terragrunt output -raw api_gateway_url 2>/dev/null || echo ""); \
-		if [ -z "$$API_GATEWAY_URL" ]; then \
-			echo "$(YELLOW)βœ— Could not get API Gateway URL from Terragrunt$(NC)"; \
-			echo "$(YELLOW)  Please set manually: make perf-test-echo API_GATEWAY_URL=https://xxx.execute-api.ca-central-1.amazonaws.com$(NC)"; \
+		API_URL=$$(cd $(SANDBOX_ROOT)/slack-api-gateway && terragrunt output -raw api_gateway_url 2>/dev/null); \
+		if [ -z "$$API_URL" ]; then \
+			echo "$(YELLOW)βœ— Could not get API Gateway URL$(NC)"; \
+			echo "$(YELLOW)  Set manually: API_GATEWAY_URL=https://xxx...$(NC)"; \
 			exit 1; \
 		fi; \
-		echo "$(GREEN)βœ“ API Gateway URL: $$API_GATEWAY_URL$(NC)"; \
-		export ENVIRONMENT=$(ENVIRONMENT) AWS_REGION=$(REGION); \
-		cd performance-tests && artillery run artillery-echo-light.yml --output results/echo-test-$$(date +%Y%m%d-%H%M%S).json; \
 	else \
-		export ENVIRONMENT=$(ENVIRONMENT) AWS_REGION=$(REGION); \
-		cd performance-tests && artillery run artillery-echo-light.yml --output results/echo-test-$$(date +%Y%m%d-%H%M%S).json; \
-	fi
-
-perf-test-quick: perf-test-install
-	@echo "$(BLUE)Running quick Artillery test (2 minutes)...$(NC)"
-	@export ENVIRONMENT=$(ENVIRONMENT) AWS_REGION=$(REGION); \
-	if [ -z "$(API_GATEWAY_URL)" ]; then \
-		export API_GATEWAY_URL=$$(cd $(SANDBOX_ROOT)/slack-api-gateway && terragrunt output -raw api_gateway_url 2>/dev/null || echo "https://placeholder.execute-api.ca-central-1.amazonaws.com"); \
+		API_URL=$(API_GATEWAY_URL); \
 	fi; \
-	cd performance-tests && artillery quick --count 100 --num 10 $$API_GATEWAY_URL/slack
-
-perf-test-results:
-	@echo "$(BLUE)Viewing latest test results...$(NC)"
-	@cd performance-tests && ./view-results.sh
-
-perf-test-report:
-	@echo "$(BLUE)Generating HTML report from latest test...$(NC)"
-	@REPORT_JSON=$${REPORT_JSON:-$$(ls -t performance-tests/results/*-test-*.json 2>/dev/null | grep -v '\.metrics\.json' | head -n1)}; \
-	if [ -z "$$REPORT_JSON" ]; then \
-		echo "$(YELLOW)βœ— No test results found$(NC)"; \
-		exit 1; \
-	fi; \
-	REPORT_HTML=$${REPORT_HTML:-$${REPORT_JSON%.json}.html}; \
-	node performance-tests/render-report.js $$REPORT_JSON $$REPORT_HTML; \
-	echo "$(GREEN)βœ“ Report written: $$REPORT_HTML$(NC)"
-
-perf-test-summary:
-	@echo "$(BLUE)Quick summary of latest test...$(NC)"
-	@LATEST=$$(ls -t performance-tests/results/*-test-*.json 2>/dev/null | head -n1); \
-	if [ -z "$$LATEST" ]; then \
-		echo "$(YELLOW)βœ— No test results found$(NC)"; \
-		exit 1; \
-	fi; \
-	echo "$(GREEN)File: $$LATEST$(NC)"; \
-	echo ""; \
-	jq -r '"Requests: " + (.aggregate.counters["http.requests"] | tostring), \
-	        "P50: " + (.aggregate.summaries["http.response_time"].median | tostring) + "ms", \
-	        "P95: " + (.aggregate.summaries["http.response_time"].p95 | tostring) + "ms", \
-	        "P99: " + (.aggregate.summaries["http.response_time"].p99 | tostring) + "ms", \
-	        "Errors: " + ((.aggregate.counters["errors.total"] // 0) | tostring)' \
-	        $$LATEST 2>/dev/null || cat $$LATEST | grep -A 20 "aggregate"
+	CONFIG=$(PERF_CONFIG_$(PERF_PROFILE)); \
+	export API_GATEWAY_URL=$$API_URL ENVIRONMENT=$(ENVIRONMENT) AWS_REGION=$(REGION); \
+	cd performance-tests && artillery run $$CONFIG --output results/test-$$(date +%Y%m%d-%H%M%S).json
 
-perf-test-install:
-	@echo "$(BLUE)Checking Artillery installation...$(NC)"
-	@if ! command -v artillery > /dev/null; then \
-		echo "$(YELLOW)Installing Artillery globally...$(NC)"; \
-		npm install -g artillery artillery-plugin-metrics-by-endpoint; \
-	fi
-	@echo "$(BLUE)Installing test dependencies...$(NC)"
-	@cd performance-tests && npm install --no-save @aws-sdk/client-ssm 2>/dev/null || true
-	@echo "$(GREEN)βœ“ Artillery ready$(NC)"
-
-perf-test-clean:
-	@echo "$(BLUE)Cleaning performance test results...$(NC)"
-	rm -rf performance-tests/results/*.json
-	rm -rf performance-tests/results/*.html
-	@echo "$(GREEN)βœ“ Performance test results cleaned$(NC)"
-
-perf-test-analyze:
-	@echo "$(BLUE)Analyzing performance test results from CloudWatch...$(NC)"
-	@cd performance-tests && ./analyze-performance.sh 15
-perf-test-analyze-test:
-	@echo "$(BLUE)Analyzing from latest Artillery test result...$(NC)"
+perf-analyze:
+	@echo "$(BLUE)Analyzing latest test results...$(NC)"
 	@cd performance-tests && ./analyze-performance.sh --from-test
 
-perf-test-analyze-test-quiet:
-	@echo "$(BLUE)Analyzing from latest Artillery test result (quiet mode)...$(NC)"
-	@cd performance-tests && ./analyze-performance.sh --from-test --quiet 2>/dev/null
+perf-analyze-quiet:
+	@cd performance-tests && ./analyze-e2e-json.sh 2>&1 | grep -E '(βœ“|Analyzing|Error)' || true
 
-perf-test-json:
-	@echo "$(BLUE)Exporting E2E metrics to JSON...$(NC)"
-	@cd performance-tests && ./analyze-e2e-json.sh >/dev/null
-	@echo "$(GREEN)βœ“ Metrics file created in performance-tests/results/$(NC)"
\ No newline at end of file
+perf-summary:
+	@LATEST=$$(ls -t performance-tests/results/*.json 2>/dev/null | grep -v '\.metrics\.json' | head -n1); \
+	if [ -z "$$LATEST" ]; then echo "$(YELLOW)No results$(NC)"; exit 1; fi; \
+	echo "$(GREEN)$$LATEST$(NC)"; \
+	jq -r '"Requests: " + (.aggregate.counters["http.requests"] | tostring), \
+	       "P50: " + (.aggregate.summaries["http.response_time"].median | tostring) + "ms", \
+	       "P95: " + (.aggregate.summaries["http.response_time"].p95 | tostring) + "ms", \
+	       "Errors: " + ((.aggregate.counters["errors.total"] // 0) | tostring)' $$LATEST
+
+perf-report:
+	@LATEST=$$(ls -t performance-tests/results/*.json 2>/dev/null | grep -v '\.metrics\.json' | head -n1); \
+	if [ -z "$$LATEST" ]; then echo "$(YELLOW)No results$(NC)"; exit 1; fi; \
+	REPORT=$${LATEST%.json}.html; \
+	node performance-tests/render-report.js $$LATEST $$REPORT; \
+	echo "$(GREEN)βœ“ Report: $$REPORT$(NC)"
+
+perf-clean:
+	@rm -rf performance-tests/results/*.json performance-tests/results/*.html
+	@echo "$(GREEN)βœ“ Cleaned$(NC)"
\ No newline at end of file
diff --git a/applications/chatops/slack-bot/performance-tests/analyze-performance.sh b/applications/chatops/slack-bot/performance-tests/analyze-performance.sh
index df51596..d29fd45 100755
--- a/applications/chatops/slack-bot/performance-tests/analyze-performance.sh
+++ b/applications/chatops/slack-bot/performance-tests/analyze-performance.sh
@@ -292,10 +292,15 @@ WORKER_COLD_ID=$(cat /tmp/worker-cold.json | jq -r '.queryId')
 sleep 5
 aws logs get-query-results --query-id ${WORKER_COLD_ID} --region ${REGION} --output table
 
-echo ""
-echo "========================================"
-echo "CloudWatch Metrics (Lambda)"
-echo "========================================"
+echo_info ""
+echo_info "========================================"
+echo_info "CloudWatch Metrics (Lambda)"
+echo_info "========================================"
+
+# Skip CloudWatch Metrics in quiet mode (optional, can fail)
+if [ "$QUIET_MODE" = true ]; then
+  echo_info "(Skipped in quiet mode)"
+else
 
 # 5. Concurrent Executions
 echo_info ""
@@ -392,6 +397,8 @@ aws cloudwatch get-metric-statistics \
   --region ${REGION} \
   --output table
 
+fi  # End of quiet mode skip
+
 echo_info ""
 echo_info "========================================"
 echo_info "Component-Level Breakdown"
diff --git a/applications/chatops/slack-bot/performance-tests/artillery-echo-minimal.yml b/applications/chatops/slack-bot/performance-tests/artillery-echo-minimal.yml
new file mode 100644
index 0000000..5ffb318
--- /dev/null
+++ b/applications/chatops/slack-bot/performance-tests/artillery-echo-minimal.yml
@@ -0,0 +1,49 @@
+config:
+  target: "{{ $processEnvironment.API_GATEWAY_URL }}"
+
+  phases:
+    # Cold start μΈ‘μ • (1 request)
+    - duration: 5
+      arrivalRate: 1
+      name: "Cold start check"
+
+    # Warm-up (ν†΅κ³„μ μœΌλ‘œ μœ μ˜λ―Έν•œ μƒ˜ν”Œ: 50개)
+    - duration: 25
+      arrivalRate: 2
+      name: "Warm state baseline (2 req/s)"
+
+    # Peak load test (짧게 λΆ€ν•˜ 확인: 30개)
+    - duration: 10
+      arrivalRate: 3
+      rampTo: 6
+      name: "Peak load spike (3 -> 6 req/s)"
+
+    # Sustained performance (μ•ˆμ • μƒνƒœ 확인: 40개)
+    - duration: 20
+      arrivalRate: 2
+      name: "Sustained performance (2 req/s)"
+
+  processor: "./slack-signature-processor.js"
+
+  plugins:
+    metrics-by-endpoint:
+      stripQueryString: true
+      metricsNamespace: "slack_bot_echo_minimal"
+
+  # Performance thresholds
+  ensure:
+    maxErrorRate: 1     # Max 1% error rate
+    p95: 3000           # 95th percentile under 3s
+    p99: 4000           # 99th percentile under 4s
+
+scenarios:
+  # Minimal echo command test - λͺ¨λ“  λ©”νŠΈλ¦­ 확인 κ°€λŠ₯
+  - name: "Echo Command Minimal Test"
+    flow:
+      - post:
+          url: "/"
+          headers:
+            Content-Type: "application/x-www-form-urlencoded"
+          beforeRequest: "generateSlackSignature"
+          body: ""  # Body will be generated in beforeRequest hook
+          afterResponse: "captureMetrics"

From 43d39e2aed68920ecfaf2a7e84bcc90daf389f85 Mon Sep 17 00:00:00 2001
From: Hyunseok Seo 
Date: Fri, 2 Jan 2026 13:09:48 -0500
Subject: [PATCH 07/15] Add performance test mock for Slack response URL
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Problem:
- Performance tests use fake Slack URLs
- Fake URLs return 404 errors
- Lambda fails before logging complete performance metrics

Solution:
- Use special mock URL: /test/perf-test-mock
- Worker detects and skips Slack API call for this URL
- Lambda completes successfully with full metrics

Changes:
- slack-signature-processor.js: Generate mock URL for tests
- slack-client.ts: Skip API call if URL contains /test/perf-test-mock
- No Lambda environment variable changes needed
- Real Slack URLs unaffected

Benefits:
- All e2e metrics logged: totalE2eMs, queueWaitMs, syncResponseMs, asyncResponseMs
- DLQ no longer fills with test failures
- Performance tests now generate complete data

πŸ€– Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude 
---
 .../performance-tests/slack-signature-processor.js       | 3 ++-
 .../chatops/slack-bot/src/shared/slack-client.ts         | 9 +++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/applications/chatops/slack-bot/performance-tests/slack-signature-processor.js b/applications/chatops/slack-bot/performance-tests/slack-signature-processor.js
index 3a64df1..427ef7f 100644
--- a/applications/chatops/slack-bot/performance-tests/slack-signature-processor.js
+++ b/applications/chatops/slack-bot/performance-tests/slack-signature-processor.js
@@ -65,7 +65,8 @@ function generateSlackSignature(requestParams, context, ee, next) {
   const messageId = Math.floor(Math.random() * 100000) + 1;
 
   // Build body with actual values (not templates)
-  const body = `token=test&team_id=T123&team_domain=test&channel_id=C123&channel_name=general&user_id=U${userId}&user_name=testuser${userName}&command=/echo&text=performance test message ${messageId}&response_url=https://hooks.slack.com/commands/T123/456/token&trigger_id=123.456.abc`;
+  // Use special response_url for performance tests (will be mocked in Lambda)
+  const body = `token=test&team_id=T123&team_domain=test&channel_id=C123&channel_name=general&user_id=U${userId}&user_name=testuser${userName}&command=/echo&text=performance test message ${messageId}&response_url=https://hooks.slack.com/test/perf-test-mock&trigger_id=123.456.abc`;
 
   // Replace requestParams.body with our pre-evaluated body
   requestParams.body = body;
diff --git a/applications/chatops/slack-bot/src/shared/slack-client.ts b/applications/chatops/slack-bot/src/shared/slack-client.ts
index 632814a..e44c565 100644
--- a/applications/chatops/slack-bot/src/shared/slack-client.ts
+++ b/applications/chatops/slack-bot/src/shared/slack-client.ts
@@ -11,6 +11,15 @@ export async function sendSlackResponse(
   try {
     logger.debug('Sending Slack response', { responseUrl, response });
 
+    // Skip Slack API call for performance test mock URL
+    // Performance tests use special URL to avoid 404 errors
+    if (responseUrl.includes('/test/perf-test-mock')) {
+      logger.info('Slack response skipped (performance test mode)', {
+        responseType: response.response_type
+      });
+      return;
+    }
+
     await axios.post(responseUrl, response, {
       headers: {
         'Content-Type': 'application/json'

From 1cf38f3b3d7797b07de851aa36f12f8418e8a53f Mon Sep 17 00:00:00 2001
From: Hyunseok Seo 
Date: Fri, 2 Jan 2026 16:20:18 -0500
Subject: [PATCH 08/15] Improve performance report UI and remove test delay
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Enhanced E2E metrics visualization with highlighted cards
- Reorganized report layout: E2E metrics β†’ Component breakdown β†’ Service metrics
- Added responsive design for mobile/tablet (768px, 480px breakpoints)
- Implemented consistent color palette across all charts
- Added E2E Timeline breakdown chart with stacked bars and percentage display
- Improved chart tooltips with 'index' mode for better UX
- Added Service Latency Distribution chart with percentile comparison
- Removed MAX column from comparison (data not available for E2E)
- Added note about independent percentile calculations
- Created E2E Component Details table showing time breakdown
- Improved chart interaction: larger point radius, hover effects
- Removed 2000ms artificial delay from echo worker for realistic testing

πŸ€– Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude 
---
 .../performance-tests/render-report.js        | 502 ++++++++++++++++--
 .../slack-bot/src/workers/echo/index.ts       |   3 -
 2 files changed, 446 insertions(+), 59 deletions(-)

diff --git a/applications/chatops/slack-bot/performance-tests/render-report.js b/applications/chatops/slack-bot/performance-tests/render-report.js
index 6cff8a8..08339f2 100644
--- a/applications/chatops/slack-bot/performance-tests/render-report.js
+++ b/applications/chatops/slack-bot/performance-tests/render-report.js
@@ -254,10 +254,20 @@ const html = `
       }
       .cards {
         display: grid;
-        grid-template-columns: repeat(auto-fit, minmax(180px, 1fr));
+        grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
         gap: 12px;
         margin-bottom: 24px;
       }
+      @media (max-width: 768px) {
+        .cards {
+          grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
+        }
+      }
+      @media (max-width: 480px) {
+        .cards {
+          grid-template-columns: 1fr;
+        }
+      }
       .card {
         background: var(--card);
         border: 1px solid var(--border);
@@ -287,9 +297,19 @@ const html = `
       }
       .grid {
         display: grid;
-        grid-template-columns: repeat(auto-fit, minmax(320px, 1fr));
+        grid-template-columns: repeat(auto-fit, minmax(400px, 1fr));
         gap: 16px;
       }
+      @media (max-width: 1200px) {
+        .grid {
+          grid-template-columns: repeat(auto-fit, minmax(320px, 1fr));
+        }
+      }
+      @media (max-width: 768px) {
+        .grid {
+          grid-template-columns: 1fr;
+        }
+      }
       .panel {
         background: var(--card);
         border: 1px solid var(--border);
@@ -499,42 +519,123 @@ const html = `
       
 
       
-